Use Off-Policy VPG to Play CartPole-v0

TensorFlow version

%matplotlib inline

import sys
import logging
import itertools

import numpy as np
np.random.seed(0)
import pandas as pd
import gym
import matplotlib.pyplot as plt
import tensorflow.compat.v2 as tf
tf.random.set_seed(0)
from tensorflow import keras
from tensorflow import nn
from tensorflow import optimizers
from tensorflow.keras import layers
from tensorflow.keras import losses

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')

env = gym.make('CartPole-v0')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

11:29:13 [INFO] env: <CartPoleEnv<CartPole-v0>>
11:29:13 [INFO] action_space: Discrete(2)
11:29:13 [INFO] observation_space: Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)
11:29:13 [INFO] reward_range: (-inf, inf)
11:29:13 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
11:29:13 [INFO] _max_episode_steps: 200
11:29:13 [INFO] _elapsed_steps: None
11:29:13 [INFO] id: CartPole-v0
11:29:13 [INFO] entry_point: gym.envs.classic_control:CartPoleEnv
11:29:13 [INFO] reward_threshold: 195.0
11:29:13 [INFO] nondeterministic: False
11:29:13 [INFO] max_episode_steps: 200
11:29:13 [INFO] _kwargs: {}
11:29:13 [INFO] _env_name: CartPole

class OffPolicyVPGAgent:
    def __init__(self, env):
        self.action_n = env.action_space.n
        self.gamma = 0.99

        def dot(y_true, y_pred):
            return -tf.reduce_sum(y_true * y_pred, axis=-1)

        self.policy_net = self.build_net(hidden_sizes=[],
                output_size=self.action_n,
                output_activation=nn.softmax,
                loss=dot, learning_rate=0.06)

    def build_net(self, hidden_sizes, output_size,
            activation=nn.relu, output_activation=None,
            use_bias=False, loss=losses.mse, learning_rate=0.001):
        model = keras.Sequential()
        for hidden_size in hidden_sizes:
            model.add(layers.Dense(units=hidden_size,
                    activation=activation, use_bias=use_bias))
        model.add(layers.Dense(units=output_size,
                activation=output_activation, use_bias=use_bias))
        optimizer = optimizers.Adam(learning_rate)
        model.compile(optimizer=optimizer, loss=loss)
        return model

    def reset(self, mode=None):
        self.mode = mode
        if self.mode == 'train':
            self.trajectory = []

    def step(self, observation, reward, terminated):
        if self.mode == 'train':
            action = np.random.choice(self.action_n)  # use random policy
            self.trajectory += [observation, reward, terminated, action]
        else:
            probs = self.policy_net.predict(observation[np.newaxis], verbose=0)[0]
            action = np.random.choice(self.action_n, p=probs)
        return action

    def close(self):
        if self.mode == 'train':
            self.learn()

    def learn(self):
        df = pd.DataFrame(np.array(self.trajectory, dtype=object).reshape(-1, 4),
                columns=['state', 'reward', 'terminated', 'action'])
        df['discount'] = self.gamma ** df.index.to_series()
        df['discounted_reward'] = df['discount'] * df['reward'].astype(float)
        df['discounted_return'] = df['discounted_reward'][::-1].cumsum()
        states = np.stack(df['state'])
        actions = np.eye(self.action_n)[df['action'].astype(int)]
        df['behavior_prob'] = 1. / self.action_n
        df['sample_weight'] = df['discounted_return'] / df['behavior_prob']
        sample_weight = df[['sample_weight',]].values
        self.policy_net.fit(states, actions, sample_weight=sample_weight,
                verbose=0)


agent = OffPolicyVPGAgent(env)

def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== train ====')
episode_rewards = []
for episode in itertools.count():
    play_episode(env, agent, seed=episode,
            mode='train')
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('train episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > 195:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

11:29:14 [INFO] ==== train ====
11:29:15 [INFO] NumExpr defaulting to 4 threads.
11:29:20 [INFO] train episode 0: reward = 34.00, steps = 34
11:29:21 [INFO] train episode 1: reward = 16.00, steps = 16
11:29:22 [INFO] train episode 2: reward = 20.00, steps = 20
11:29:23 [INFO] train episode 3: reward = 17.00, steps = 17
11:29:25 [INFO] train episode 4: reward = 26.00, steps = 26
11:29:25 [INFO] train episode 5: reward = 12.00, steps = 12
11:29:27 [INFO] train episode 6: reward = 25.00, steps = 25
11:29:28 [INFO] train episode 7: reward = 19.00, steps = 19
11:29:30 [INFO] train episode 8: reward = 37.00, steps = 37
11:29:31 [INFO] train episode 9: reward = 27.00, steps = 27
11:29:32 [INFO] train episode 10: reward = 16.00, steps = 16
11:29:33 [INFO] train episode 11: reward = 23.00, steps = 23
11:29:34 [INFO] train episode 12: reward = 12.00, steps = 12
11:29:35 [INFO] train episode 13: reward = 24.00, steps = 24
11:29:36 [INFO] train episode 14: reward = 10.00, steps = 10
11:29:37 [INFO] train episode 15: reward = 25.00, steps = 25
11:29:38 [INFO] train episode 16: reward = 25.00, steps = 25
11:29:40 [INFO] train episode 17: reward = 34.00, steps = 34
11:29:41 [INFO] train episode 18: reward = 13.00, steps = 13
11:29:42 [INFO] train episode 19: reward = 17.00, steps = 17
11:29:43 [INFO] train episode 20: reward = 11.00, steps = 11
11:29:44 [INFO] train episode 21: reward = 19.00, steps = 19
11:29:48 [INFO] train episode 22: reward = 75.00, steps = 75
11:29:49 [INFO] train episode 23: reward = 20.00, steps = 20
11:29:52 [INFO] train episode 24: reward = 55.00, steps = 55
11:29:53 [INFO] train episode 25: reward = 19.00, steps = 19
11:29:56 [INFO] train episode 26: reward = 70.00, steps = 70
11:29:56 [INFO] train episode 27: reward = 11.00, steps = 11
11:29:58 [INFO] train episode 28: reward = 24.00, steps = 24
11:30:02 [INFO] train episode 29: reward = 81.00, steps = 81
11:30:05 [INFO] train episode 30: reward = 68.00, steps = 68
11:30:10 [INFO] train episode 31: reward = 111.00, steps = 111
11:30:12 [INFO] train episode 32: reward = 44.00, steps = 44
11:30:14 [INFO] train episode 33: reward = 42.00, steps = 42
11:30:15 [INFO] train episode 34: reward = 24.00, steps = 24
11:30:18 [INFO] train episode 35: reward = 65.00, steps = 65
11:30:20 [INFO] train episode 36: reward = 46.00, steps = 46
11:30:23 [INFO] train episode 37: reward = 59.00, steps = 59
11:30:27 [INFO] train episode 38: reward = 96.00, steps = 96
11:30:30 [INFO] train episode 39: reward = 61.00, steps = 61
11:30:33 [INFO] train episode 40: reward = 49.00, steps = 49
11:30:37 [INFO] train episode 41: reward = 105.00, steps = 105
11:30:41 [INFO] train episode 42: reward = 83.00, steps = 83
11:30:46 [INFO] train episode 43: reward = 102.00, steps = 102
11:30:48 [INFO] train episode 44: reward = 41.00, steps = 41
11:30:51 [INFO] train episode 45: reward = 58.00, steps = 58
11:30:53 [INFO] train episode 46: reward = 55.00, steps = 55
11:30:56 [INFO] train episode 47: reward = 51.00, steps = 51
11:30:58 [INFO] train episode 48: reward = 55.00, steps = 55
11:31:03 [INFO] train episode 49: reward = 102.00, steps = 102
11:31:06 [INFO] train episode 50: reward = 58.00, steps = 58
11:31:09 [INFO] train episode 51: reward = 81.00, steps = 81
11:31:11 [INFO] train episode 52: reward = 24.00, steps = 24
11:31:13 [INFO] train episode 53: reward = 50.00, steps = 50
11:31:16 [INFO] train episode 54: reward = 76.00, steps = 76
11:31:19 [INFO] train episode 55: reward = 62.00, steps = 62
11:31:22 [INFO] train episode 56: reward = 51.00, steps = 51
11:31:26 [INFO] train episode 57: reward = 88.00, steps = 88
11:31:28 [INFO] train episode 58: reward = 40.00, steps = 40
11:31:31 [INFO] train episode 59: reward = 71.00, steps = 71
11:31:34 [INFO] train episode 60: reward = 70.00, steps = 70
11:31:36 [INFO] train episode 61: reward = 39.00, steps = 39
11:31:38 [INFO] train episode 62: reward = 51.00, steps = 51
11:31:41 [INFO] train episode 63: reward = 56.00, steps = 56
11:31:44 [INFO] train episode 64: reward = 54.00, steps = 54
11:31:47 [INFO] train episode 65: reward = 70.00, steps = 70
11:31:50 [INFO] train episode 66: reward = 68.00, steps = 68
11:31:53 [INFO] train episode 67: reward = 61.00, steps = 61
11:31:55 [INFO] train episode 68: reward = 43.00, steps = 43
11:31:57 [INFO] train episode 69: reward = 51.00, steps = 51
11:32:00 [INFO] train episode 70: reward = 58.00, steps = 58
11:32:03 [INFO] train episode 71: reward = 64.00, steps = 64
11:32:05 [INFO] train episode 72: reward = 38.00, steps = 38
11:32:07 [INFO] train episode 73: reward = 34.00, steps = 34
11:32:11 [INFO] train episode 74: reward = 88.00, steps = 88
11:32:16 [INFO] train episode 75: reward = 125.00, steps = 125
11:32:21 [INFO] train episode 76: reward = 92.00, steps = 92
11:32:25 [INFO] train episode 77: reward = 103.00, steps = 103
11:32:29 [INFO] train episode 78: reward = 82.00, steps = 82
11:32:31 [INFO] train episode 79: reward = 37.00, steps = 37
11:32:33 [INFO] train episode 80: reward = 38.00, steps = 38
11:32:36 [INFO] train episode 81: reward = 61.00, steps = 61
11:32:38 [INFO] train episode 82: reward = 60.00, steps = 60
11:32:41 [INFO] train episode 83: reward = 63.00, steps = 63
11:32:45 [INFO] train episode 84: reward = 81.00, steps = 81
11:32:48 [INFO] train episode 85: reward = 54.00, steps = 54
11:32:50 [INFO] train episode 86: reward = 50.00, steps = 50
11:32:53 [INFO] train episode 87: reward = 72.00, steps = 72
11:32:56 [INFO] train episode 88: reward = 59.00, steps = 59
11:33:01 [INFO] train episode 89: reward = 96.00, steps = 96
11:33:03 [INFO] train episode 90: reward = 48.00, steps = 48
11:33:07 [INFO] train episode 91: reward = 83.00, steps = 83
11:33:09 [INFO] train episode 92: reward = 50.00, steps = 50
11:33:12 [INFO] train episode 93: reward = 54.00, steps = 54
11:33:14 [INFO] train episode 94: reward = 47.00, steps = 47
11:33:16 [INFO] train episode 95: reward = 45.00, steps = 45
11:33:19 [INFO] train episode 96: reward = 68.00, steps = 68
11:33:22 [INFO] train episode 97: reward = 66.00, steps = 66
11:33:25 [INFO] train episode 98: reward = 53.00, steps = 53
11:33:28 [INFO] train episode 99: reward = 58.00, steps = 58
11:33:32 [INFO] train episode 100: reward = 90.00, steps = 90
11:33:35 [INFO] train episode 101: reward = 74.00, steps = 74
11:33:38 [INFO] train episode 102: reward = 61.00, steps = 61
11:33:41 [INFO] train episode 103: reward = 50.00, steps = 50
11:33:43 [INFO] train episode 104: reward = 51.00, steps = 51
11:33:46 [INFO] train episode 105: reward = 58.00, steps = 58
11:33:50 [INFO] train episode 106: reward = 100.00, steps = 100
11:33:53 [INFO] train episode 107: reward = 49.00, steps = 49
11:33:55 [INFO] train episode 108: reward = 50.00, steps = 50
11:33:57 [INFO] train episode 109: reward = 32.00, steps = 32
11:34:01 [INFO] train episode 110: reward = 81.00, steps = 81
11:34:04 [INFO] train episode 111: reward = 75.00, steps = 75
11:34:08 [INFO] train episode 112: reward = 76.00, steps = 76
11:34:11 [INFO] train episode 113: reward = 71.00, steps = 71
11:34:14 [INFO] train episode 114: reward = 72.00, steps = 72
11:34:17 [INFO] train episode 115: reward = 63.00, steps = 63
11:34:21 [INFO] train episode 116: reward = 87.00, steps = 87
11:34:26 [INFO] train episode 117: reward = 104.00, steps = 104
11:34:30 [INFO] train episode 118: reward = 91.00, steps = 91
11:34:33 [INFO] train episode 119: reward = 53.00, steps = 53
11:34:36 [INFO] train episode 120: reward = 61.00, steps = 61
11:34:39 [INFO] train episode 121: reward = 61.00, steps = 61
11:34:41 [INFO] train episode 122: reward = 57.00, steps = 57
11:34:46 [INFO] train episode 123: reward = 97.00, steps = 97
11:34:50 [INFO] train episode 124: reward = 70.00, steps = 70
11:34:52 [INFO] train episode 125: reward = 41.00, steps = 41
11:34:55 [INFO] train episode 126: reward = 66.00, steps = 66
11:34:58 [INFO] train episode 127: reward = 74.00, steps = 74
11:35:02 [INFO] train episode 128: reward = 84.00, steps = 84
11:35:05 [INFO] train episode 129: reward = 59.00, steps = 59
11:35:07 [INFO] train episode 130: reward = 53.00, steps = 53
11:35:14 [INFO] train episode 131: reward = 145.00, steps = 145
11:35:18 [INFO] train episode 132: reward = 93.00, steps = 93
11:35:22 [INFO] train episode 133: reward = 85.00, steps = 85
11:35:26 [INFO] train episode 134: reward = 81.00, steps = 81
11:35:28 [INFO] train episode 135: reward = 40.00, steps = 40
11:35:31 [INFO] train episode 136: reward = 56.00, steps = 56
11:35:33 [INFO] train episode 137: reward = 59.00, steps = 59
11:35:36 [INFO] train episode 138: reward = 52.00, steps = 52
11:35:38 [INFO] train episode 139: reward = 39.00, steps = 39
11:35:45 [INFO] train episode 140: reward = 143.00, steps = 143
11:35:50 [INFO] train episode 141: reward = 104.00, steps = 104
11:35:53 [INFO] train episode 142: reward = 65.00, steps = 65
11:35:55 [INFO] train episode 143: reward = 50.00, steps = 50
11:35:58 [INFO] train episode 144: reward = 72.00, steps = 72
11:36:02 [INFO] train episode 145: reward = 67.00, steps = 67
11:36:07 [INFO] train episode 146: reward = 127.00, steps = 127
11:36:16 [INFO] train episode 147: reward = 188.00, steps = 188
11:36:20 [INFO] train episode 148: reward = 90.00, steps = 90
11:36:23 [INFO] train episode 149: reward = 66.00, steps = 66
11:36:26 [INFO] train episode 150: reward = 71.00, steps = 71
11:36:30 [INFO] train episode 151: reward = 71.00, steps = 71
11:36:33 [INFO] train episode 152: reward = 81.00, steps = 81
11:36:37 [INFO] train episode 153: reward = 72.00, steps = 72
11:36:40 [INFO] train episode 154: reward = 68.00, steps = 68
11:36:46 [INFO] train episode 155: reward = 126.00, steps = 126
11:36:47 [INFO] train episode 156: reward = 38.00, steps = 38
11:36:50 [INFO] train episode 157: reward = 60.00, steps = 60
11:36:53 [INFO] train episode 158: reward = 68.00, steps = 68
11:36:58 [INFO] train episode 159: reward = 89.00, steps = 89
11:37:02 [INFO] train episode 160: reward = 107.00, steps = 107
11:37:08 [INFO] train episode 161: reward = 122.00, steps = 122
11:37:10 [INFO] train episode 162: reward = 49.00, steps = 49
11:37:14 [INFO] train episode 163: reward = 71.00, steps = 71
11:37:16 [INFO] train episode 164: reward = 48.00, steps = 48
11:37:21 [INFO] train episode 165: reward = 125.00, steps = 125
11:37:26 [INFO] train episode 166: reward = 110.00, steps = 110
11:37:30 [INFO] train episode 167: reward = 85.00, steps = 85
11:37:34 [INFO] train episode 168: reward = 86.00, steps = 86
11:37:38 [INFO] train episode 169: reward = 75.00, steps = 75
11:37:41 [INFO] train episode 170: reward = 63.00, steps = 63
11:37:44 [INFO] train episode 171: reward = 74.00, steps = 74
11:37:48 [INFO] train episode 172: reward = 78.00, steps = 78
11:37:51 [INFO] train episode 173: reward = 64.00, steps = 64
11:37:52 [INFO] train episode 174: reward = 39.00, steps = 39
11:37:56 [INFO] train episode 175: reward = 69.00, steps = 69
11:37:59 [INFO] train episode 176: reward = 73.00, steps = 73
11:38:03 [INFO] train episode 177: reward = 80.00, steps = 80
11:38:05 [INFO] train episode 178: reward = 56.00, steps = 56
11:38:07 [INFO] train episode 179: reward = 43.00, steps = 43
11:38:12 [INFO] train episode 180: reward = 100.00, steps = 100
11:38:15 [INFO] train episode 181: reward = 67.00, steps = 67
11:38:17 [INFO] train episode 182: reward = 46.00, steps = 46
11:38:21 [INFO] train episode 183: reward = 77.00, steps = 77
11:38:24 [INFO] train episode 184: reward = 70.00, steps = 70
11:38:27 [INFO] train episode 185: reward = 58.00, steps = 58
11:38:29 [INFO] train episode 186: reward = 55.00, steps = 55
11:38:34 [INFO] train episode 187: reward = 96.00, steps = 96
11:38:40 [INFO] train episode 188: reward = 138.00, steps = 138
11:38:42 [INFO] train episode 189: reward = 52.00, steps = 52
11:38:47 [INFO] train episode 190: reward = 91.00, steps = 91
11:38:50 [INFO] train episode 191: reward = 70.00, steps = 70
11:38:53 [INFO] train episode 192: reward = 73.00, steps = 73
11:38:57 [INFO] train episode 193: reward = 81.00, steps = 81
11:39:03 [INFO] train episode 194: reward = 139.00, steps = 139
11:39:06 [INFO] train episode 195: reward = 63.00, steps = 63
11:39:13 [INFO] train episode 196: reward = 132.00, steps = 132
11:39:17 [INFO] train episode 197: reward = 82.00, steps = 82
11:39:20 [INFO] train episode 198: reward = 81.00, steps = 81
11:39:24 [INFO] train episode 199: reward = 78.00, steps = 78
11:39:28 [INFO] train episode 200: reward = 93.00, steps = 93
11:39:31 [INFO] train episode 201: reward = 70.00, steps = 70
11:39:34 [INFO] train episode 202: reward = 63.00, steps = 63
11:39:39 [INFO] train episode 203: reward = 95.00, steps = 95
11:39:42 [INFO] train episode 204: reward = 70.00, steps = 70
11:39:47 [INFO] train episode 205: reward = 104.00, steps = 104
11:39:50 [INFO] train episode 206: reward = 66.00, steps = 66
11:39:57 [INFO] train episode 207: reward = 160.00, steps = 160
11:40:01 [INFO] train episode 208: reward = 95.00, steps = 95
11:40:04 [INFO] train episode 209: reward = 54.00, steps = 54
11:40:06 [INFO] train episode 210: reward = 53.00, steps = 53
11:40:09 [INFO] train episode 211: reward = 58.00, steps = 58
11:40:13 [INFO] train episode 212: reward = 95.00, steps = 95
11:40:18 [INFO] train episode 213: reward = 89.00, steps = 89
11:40:23 [INFO] train episode 214: reward = 115.00, steps = 115
11:40:28 [INFO] train episode 215: reward = 112.00, steps = 112
11:40:32 [INFO] train episode 216: reward = 92.00, steps = 92
11:40:37 [INFO] train episode 217: reward = 104.00, steps = 104
11:40:40 [INFO] train episode 218: reward = 71.00, steps = 71
11:40:45 [INFO] train episode 219: reward = 104.00, steps = 104
11:40:48 [INFO] train episode 220: reward = 67.00, steps = 67
11:40:51 [INFO] train episode 221: reward = 68.00, steps = 68
11:40:56 [INFO] train episode 222: reward = 100.00, steps = 100
11:40:59 [INFO] train episode 223: reward = 63.00, steps = 63
11:41:01 [INFO] train episode 224: reward = 44.00, steps = 44
11:41:03 [INFO] train episode 225: reward = 51.00, steps = 51
11:41:06 [INFO] train episode 226: reward = 63.00, steps = 63
11:41:10 [INFO] train episode 227: reward = 83.00, steps = 83
11:41:16 [INFO] train episode 228: reward = 122.00, steps = 122
11:41:18 [INFO] train episode 229: reward = 60.00, steps = 60
11:41:21 [INFO] train episode 230: reward = 66.00, steps = 66
11:41:25 [INFO] train episode 231: reward = 82.00, steps = 82
11:41:29 [INFO] train episode 232: reward = 75.00, steps = 75
11:41:32 [INFO] train episode 233: reward = 67.00, steps = 67
11:41:35 [INFO] train episode 234: reward = 77.00, steps = 77
11:41:39 [INFO] train episode 235: reward = 74.00, steps = 74
11:41:44 [INFO] train episode 236: reward = 113.00, steps = 113
11:41:47 [INFO] train episode 237: reward = 66.00, steps = 66
11:41:51 [INFO] train episode 238: reward = 83.00, steps = 83
11:41:54 [INFO] train episode 239: reward = 66.00, steps = 66
11:41:57 [INFO] train episode 240: reward = 69.00, steps = 69
11:42:02 [INFO] train episode 241: reward = 101.00, steps = 101
11:42:08 [INFO] train episode 242: reward = 133.00, steps = 133
11:42:11 [INFO] train episode 243: reward = 72.00, steps = 72
11:42:17 [INFO] train episode 244: reward = 120.00, steps = 120
11:42:21 [INFO] train episode 245: reward = 104.00, steps = 104
11:42:26 [INFO] train episode 246: reward = 97.00, steps = 97
11:42:29 [INFO] train episode 247: reward = 73.00, steps = 73
11:42:32 [INFO] train episode 248: reward = 55.00, steps = 55
11:42:34 [INFO] train episode 249: reward = 62.00, steps = 62
11:42:39 [INFO] train episode 250: reward = 88.00, steps = 88
11:42:42 [INFO] train episode 251: reward = 73.00, steps = 73
11:42:44 [INFO] train episode 252: reward = 50.00, steps = 50
11:42:48 [INFO] train episode 253: reward = 79.00, steps = 79
11:42:52 [INFO] train episode 254: reward = 88.00, steps = 88
11:42:57 [INFO] train episode 255: reward = 100.00, steps = 100
11:43:04 [INFO] train episode 256: reward = 174.00, steps = 174
11:43:10 [INFO] train episode 257: reward = 121.00, steps = 121
11:43:14 [INFO] train episode 258: reward = 75.00, steps = 75
11:43:16 [INFO] train episode 259: reward = 42.00, steps = 42
11:43:17 [INFO] train episode 260: reward = 41.00, steps = 41
11:43:23 [INFO] train episode 261: reward = 123.00, steps = 123
11:43:26 [INFO] train episode 262: reward = 58.00, steps = 58
11:43:28 [INFO] train episode 263: reward = 58.00, steps = 58
11:43:33 [INFO] train episode 264: reward = 105.00, steps = 105
11:43:36 [INFO] train episode 265: reward = 61.00, steps = 61
11:43:39 [INFO] train episode 266: reward = 63.00, steps = 63
11:43:42 [INFO] train episode 267: reward = 74.00, steps = 74
11:43:46 [INFO] train episode 268: reward = 71.00, steps = 71
11:43:48 [INFO] train episode 269: reward = 56.00, steps = 56
11:43:53 [INFO] train episode 270: reward = 113.00, steps = 113
11:43:56 [INFO] train episode 271: reward = 60.00, steps = 60
11:44:00 [INFO] train episode 272: reward = 77.00, steps = 77
11:44:03 [INFO] train episode 273: reward = 67.00, steps = 67
11:44:07 [INFO] train episode 274: reward = 78.00, steps = 78
11:44:10 [INFO] train episode 275: reward = 68.00, steps = 68
11:44:14 [INFO] train episode 276: reward = 91.00, steps = 91
11:44:18 [INFO] train episode 277: reward = 84.00, steps = 84
11:44:23 [INFO] train episode 278: reward = 112.00, steps = 112
11:44:26 [INFO] train episode 279: reward = 74.00, steps = 74
11:44:31 [INFO] train episode 280: reward = 94.00, steps = 94
11:44:34 [INFO] train episode 281: reward = 75.00, steps = 75
11:44:37 [INFO] train episode 282: reward = 75.00, steps = 75
11:44:40 [INFO] train episode 283: reward = 56.00, steps = 56
11:44:44 [INFO] train episode 284: reward = 81.00, steps = 81
11:44:48 [INFO] train episode 285: reward = 79.00, steps = 79
11:44:51 [INFO] train episode 286: reward = 64.00, steps = 64
11:44:53 [INFO] train episode 287: reward = 46.00, steps = 46
11:44:56 [INFO] train episode 288: reward = 61.00, steps = 61
11:44:58 [INFO] train episode 289: reward = 52.00, steps = 52
11:45:02 [INFO] train episode 290: reward = 79.00, steps = 79
11:45:06 [INFO] train episode 291: reward = 97.00, steps = 97
11:45:09 [INFO] train episode 292: reward = 71.00, steps = 71
11:45:13 [INFO] train episode 293: reward = 81.00, steps = 81
11:45:16 [INFO] train episode 294: reward = 63.00, steps = 63
11:45:20 [INFO] train episode 295: reward = 80.00, steps = 80
11:45:24 [INFO] train episode 296: reward = 100.00, steps = 100
11:45:28 [INFO] train episode 297: reward = 80.00, steps = 80
11:45:35 [INFO] train episode 298: reward = 150.00, steps = 150
11:45:42 [INFO] train episode 299: reward = 163.00, steps = 163
11:45:44 [INFO] train episode 300: reward = 47.00, steps = 47
11:45:48 [INFO] train episode 301: reward = 86.00, steps = 86
11:45:53 [INFO] train episode 302: reward = 106.00, steps = 106
11:45:56 [INFO] train episode 303: reward = 68.00, steps = 68
11:46:00 [INFO] train episode 304: reward = 82.00, steps = 82
11:46:03 [INFO] train episode 305: reward = 66.00, steps = 66
11:46:05 [INFO] train episode 306: reward = 46.00, steps = 46
11:46:08 [INFO] train episode 307: reward = 69.00, steps = 69
11:46:11 [INFO] train episode 308: reward = 58.00, steps = 58
11:46:14 [INFO] train episode 309: reward = 60.00, steps = 60
11:46:18 [INFO] train episode 310: reward = 87.00, steps = 87
11:46:22 [INFO] train episode 311: reward = 88.00, steps = 88
11:46:27 [INFO] train episode 312: reward = 117.00, steps = 117
11:46:30 [INFO] train episode 313: reward = 52.00, steps = 52
11:46:33 [INFO] train episode 314: reward = 82.00, steps = 82
11:46:38 [INFO] train episode 315: reward = 95.00, steps = 95
11:46:42 [INFO] train episode 316: reward = 85.00, steps = 85
11:46:46 [INFO] train episode 317: reward = 86.00, steps = 86
11:46:52 [INFO] train episode 318: reward = 145.00, steps = 145
11:46:55 [INFO] train episode 319: reward = 62.00, steps = 62
11:46:59 [INFO] train episode 320: reward = 88.00, steps = 88
11:47:02 [INFO] train episode 321: reward = 53.00, steps = 53
11:47:05 [INFO] train episode 322: reward = 64.00, steps = 64
11:47:10 [INFO] train episode 323: reward = 112.00, steps = 112
11:47:13 [INFO] train episode 324: reward = 74.00, steps = 74
11:47:16 [INFO] train episode 325: reward = 58.00, steps = 58
11:47:19 [INFO] train episode 326: reward = 65.00, steps = 65
11:47:22 [INFO] train episode 327: reward = 71.00, steps = 71
11:47:26 [INFO] train episode 328: reward = 87.00, steps = 87
11:47:29 [INFO] train episode 329: reward = 61.00, steps = 61
11:47:33 [INFO] train episode 330: reward = 86.00, steps = 86
11:47:36 [INFO] train episode 331: reward = 73.00, steps = 73
11:47:40 [INFO] train episode 332: reward = 86.00, steps = 86
11:47:43 [INFO] train episode 333: reward = 54.00, steps = 54
11:47:46 [INFO] train episode 334: reward = 62.00, steps = 62
11:47:49 [INFO] train episode 335: reward = 70.00, steps = 70
11:47:52 [INFO] train episode 336: reward = 60.00, steps = 60
11:47:56 [INFO] train episode 337: reward = 82.00, steps = 82
11:48:00 [INFO] train episode 338: reward = 97.00, steps = 97
11:48:03 [INFO] train episode 339: reward = 70.00, steps = 70
11:48:07 [INFO] train episode 340: reward = 83.00, steps = 83
11:48:11 [INFO] train episode 341: reward = 91.00, steps = 91
11:48:15 [INFO] train episode 342: reward = 84.00, steps = 84
11:48:18 [INFO] train episode 343: reward = 72.00, steps = 72
11:48:24 [INFO] train episode 344: reward = 119.00, steps = 119
11:48:27 [INFO] train episode 345: reward = 59.00, steps = 59
11:48:30 [INFO] train episode 346: reward = 82.00, steps = 82
11:48:35 [INFO] train episode 347: reward = 94.00, steps = 94
11:48:39 [INFO] train episode 348: reward = 103.00, steps = 103
11:48:46 [INFO] train episode 349: reward = 149.00, steps = 149
11:48:50 [INFO] train episode 350: reward = 82.00, steps = 82
11:48:54 [INFO] train episode 351: reward = 77.00, steps = 77
11:48:57 [INFO] train episode 352: reward = 78.00, steps = 78
11:49:01 [INFO] train episode 353: reward = 75.00, steps = 75
11:49:05 [INFO] train episode 354: reward = 89.00, steps = 89
11:49:09 [INFO] train episode 355: reward = 81.00, steps = 81
11:49:15 [INFO] train episode 356: reward = 137.00, steps = 137
11:49:19 [INFO] train episode 357: reward = 105.00, steps = 105
11:49:23 [INFO] train episode 358: reward = 88.00, steps = 88
11:49:28 [INFO] train episode 359: reward = 102.00, steps = 102
11:49:32 [INFO] train episode 360: reward = 85.00, steps = 85
11:49:37 [INFO] train episode 361: reward = 106.00, steps = 106
11:49:40 [INFO] train episode 362: reward = 74.00, steps = 74
11:49:44 [INFO] train episode 363: reward = 79.00, steps = 79
11:49:48 [INFO] train episode 364: reward = 78.00, steps = 78
11:49:52 [INFO] train episode 365: reward = 93.00, steps = 93
11:49:59 [INFO] train episode 366: reward = 147.00, steps = 147
11:50:05 [INFO] train episode 367: reward = 149.00, steps = 149
11:50:12 [INFO] train episode 368: reward = 140.00, steps = 140
11:50:15 [INFO] train episode 369: reward = 71.00, steps = 71
11:50:21 [INFO] train episode 370: reward = 123.00, steps = 123
11:50:25 [INFO] train episode 371: reward = 99.00, steps = 99
11:50:28 [INFO] train episode 372: reward = 71.00, steps = 71
11:50:33 [INFO] train episode 373: reward = 110.00, steps = 110
11:50:36 [INFO] train episode 374: reward = 65.00, steps = 65
11:50:39 [INFO] train episode 375: reward = 61.00, steps = 61
11:50:44 [INFO] train episode 376: reward = 96.00, steps = 96
11:50:48 [INFO] train episode 377: reward = 100.00, steps = 100
11:50:55 [INFO] train episode 378: reward = 148.00, steps = 148
11:51:03 [INFO] train episode 379: reward = 166.00, steps = 166
11:51:05 [INFO] train episode 380: reward = 59.00, steps = 59
11:51:09 [INFO] train episode 381: reward = 84.00, steps = 84
11:51:16 [INFO] train episode 382: reward = 150.00, steps = 150
11:51:21 [INFO] train episode 383: reward = 100.00, steps = 100
11:51:23 [INFO] train episode 384: reward = 65.00, steps = 65
11:51:28 [INFO] train episode 385: reward = 92.00, steps = 92
11:51:31 [INFO] train episode 386: reward = 66.00, steps = 66
11:51:35 [INFO] train episode 387: reward = 101.00, steps = 101
11:51:39 [INFO] train episode 388: reward = 86.00, steps = 86
11:51:44 [INFO] train episode 389: reward = 92.00, steps = 92
11:51:47 [INFO] train episode 390: reward = 73.00, steps = 73
11:51:50 [INFO] train episode 391: reward = 60.00, steps = 60
11:51:53 [INFO] train episode 392: reward = 71.00, steps = 71
11:51:58 [INFO] train episode 393: reward = 109.00, steps = 109
11:52:02 [INFO] train episode 394: reward = 95.00, steps = 95
11:52:06 [INFO] train episode 395: reward = 74.00, steps = 74
11:52:11 [INFO] train episode 396: reward = 117.00, steps = 117
11:52:16 [INFO] train episode 397: reward = 110.00, steps = 110
11:52:20 [INFO] train episode 398: reward = 89.00, steps = 89
11:52:25 [INFO] train episode 399: reward = 103.00, steps = 103
11:52:30 [INFO] train episode 400: reward = 107.00, steps = 107
11:52:35 [INFO] train episode 401: reward = 111.00, steps = 111
11:52:38 [INFO] train episode 402: reward = 79.00, steps = 79
11:52:42 [INFO] train episode 403: reward = 75.00, steps = 75
11:52:46 [INFO] train episode 404: reward = 95.00, steps = 95
11:52:52 [INFO] train episode 405: reward = 133.00, steps = 133
11:52:57 [INFO] train episode 406: reward = 100.00, steps = 100
11:53:02 [INFO] train episode 407: reward = 101.00, steps = 101
11:53:06 [INFO] train episode 408: reward = 105.00, steps = 105
11:53:10 [INFO] train episode 409: reward = 88.00, steps = 88
11:53:15 [INFO] train episode 410: reward = 110.00, steps = 110
11:53:19 [INFO] train episode 411: reward = 81.00, steps = 81
11:53:25 [INFO] train episode 412: reward = 121.00, steps = 121
11:53:28 [INFO] train episode 413: reward = 76.00, steps = 76
11:53:33 [INFO] train episode 414: reward = 103.00, steps = 103
11:53:36 [INFO] train episode 415: reward = 66.00, steps = 66
11:53:40 [INFO] train episode 416: reward = 91.00, steps = 91
11:53:44 [INFO] train episode 417: reward = 83.00, steps = 83
11:53:49 [INFO] train episode 418: reward = 110.00, steps = 110
11:53:58 [INFO] train episode 419: reward = 200.00, steps = 200
11:54:02 [INFO] train episode 420: reward = 94.00, steps = 94
11:54:06 [INFO] train episode 421: reward = 76.00, steps = 76
11:54:10 [INFO] train episode 422: reward = 92.00, steps = 92
11:54:14 [INFO] train episode 423: reward = 82.00, steps = 82
11:54:16 [INFO] train episode 424: reward = 58.00, steps = 58
11:54:21 [INFO] train episode 425: reward = 99.00, steps = 99
11:54:26 [INFO] train episode 426: reward = 104.00, steps = 104
11:54:30 [INFO] train episode 427: reward = 74.00, steps = 74
11:54:32 [INFO] train episode 428: reward = 59.00, steps = 59
11:54:35 [INFO] train episode 429: reward = 58.00, steps = 58
11:54:40 [INFO] train episode 430: reward = 97.00, steps = 97
11:54:45 [INFO] train episode 431: reward = 109.00, steps = 109
11:54:51 [INFO] train episode 432: reward = 126.00, steps = 126
11:54:55 [INFO] train episode 433: reward = 88.00, steps = 88
11:54:59 [INFO] train episode 434: reward = 95.00, steps = 95
11:55:04 [INFO] train episode 435: reward = 116.00, steps = 116
11:55:08 [INFO] train episode 436: reward = 77.00, steps = 77
11:55:16 [INFO] train episode 437: reward = 172.00, steps = 172
11:55:19 [INFO] train episode 438: reward = 82.00, steps = 82
11:55:23 [INFO] train episode 439: reward = 86.00, steps = 86
11:55:29 [INFO] train episode 440: reward = 128.00, steps = 128
11:55:36 [INFO] train episode 441: reward = 142.00, steps = 142
11:55:40 [INFO] train episode 442: reward = 99.00, steps = 99
11:55:44 [INFO] train episode 443: reward = 84.00, steps = 84
11:55:50 [INFO] train episode 444: reward = 121.00, steps = 121
11:55:54 [INFO] train episode 445: reward = 95.00, steps = 95
11:55:57 [INFO] train episode 446: reward = 65.00, steps = 65
11:56:01 [INFO] train episode 447: reward = 75.00, steps = 75
11:56:05 [INFO] train episode 448: reward = 104.00, steps = 104
11:56:10 [INFO] train episode 449: reward = 100.00, steps = 100
11:56:15 [INFO] train episode 450: reward = 106.00, steps = 106
11:56:18 [INFO] train episode 451: reward = 67.00, steps = 67
11:56:23 [INFO] train episode 452: reward = 105.00, steps = 105
11:56:26 [INFO] train episode 453: reward = 63.00, steps = 63
11:56:30 [INFO] train episode 454: reward = 93.00, steps = 93
11:56:34 [INFO] train episode 455: reward = 91.00, steps = 91
11:56:39 [INFO] train episode 456: reward = 106.00, steps = 106
11:56:44 [INFO] train episode 457: reward = 114.00, steps = 114
11:56:48 [INFO] train episode 458: reward = 80.00, steps = 80
11:56:53 [INFO] train episode 459: reward = 116.00, steps = 116
11:56:56 [INFO] train episode 460: reward = 76.00, steps = 76
11:57:01 [INFO] train episode 461: reward = 97.00, steps = 97
11:57:05 [INFO] train episode 462: reward = 87.00, steps = 87
11:57:12 [INFO] train episode 463: reward = 144.00, steps = 144
11:57:15 [INFO] train episode 464: reward = 73.00, steps = 73
11:57:20 [INFO] train episode 465: reward = 118.00, steps = 118
11:57:24 [INFO] train episode 466: reward = 82.00, steps = 82
11:57:28 [INFO] train episode 467: reward = 88.00, steps = 88
11:57:34 [INFO] train episode 468: reward = 119.00, steps = 119
11:57:39 [INFO] train episode 469: reward = 116.00, steps = 116
11:57:44 [INFO] train episode 470: reward = 109.00, steps = 109
11:57:48 [INFO] train episode 471: reward = 84.00, steps = 84
11:57:57 [INFO] train episode 472: reward = 200.00, steps = 200
11:58:02 [INFO] train episode 473: reward = 107.00, steps = 107
11:58:08 [INFO] train episode 474: reward = 130.00, steps = 130
11:58:11 [INFO] train episode 475: reward = 57.00, steps = 57
11:58:13 [INFO] train episode 476: reward = 56.00, steps = 56
11:58:18 [INFO] train episode 477: reward = 100.00, steps = 100
11:58:23 [INFO] train episode 478: reward = 110.00, steps = 110
11:58:27 [INFO] train episode 479: reward = 92.00, steps = 92
11:58:31 [INFO] train episode 480: reward = 86.00, steps = 86
11:58:36 [INFO] train episode 481: reward = 121.00, steps = 121
11:58:41 [INFO] train episode 482: reward = 97.00, steps = 97
11:58:45 [INFO] train episode 483: reward = 86.00, steps = 86
11:58:51 [INFO] train episode 484: reward = 131.00, steps = 131
11:58:54 [INFO] train episode 485: reward = 65.00, steps = 65
11:58:58 [INFO] train episode 486: reward = 95.00, steps = 95
11:59:03 [INFO] train episode 487: reward = 96.00, steps = 96
11:59:06 [INFO] train episode 488: reward = 73.00, steps = 73
11:59:09 [INFO] train episode 489: reward = 66.00, steps = 66
11:59:16 [INFO] train episode 490: reward = 137.00, steps = 137
11:59:20 [INFO] train episode 491: reward = 86.00, steps = 86
11:59:23 [INFO] train episode 492: reward = 69.00, steps = 69
11:59:28 [INFO] train episode 493: reward = 119.00, steps = 119
11:59:33 [INFO] train episode 494: reward = 112.00, steps = 112
11:59:37 [INFO] train episode 495: reward = 89.00, steps = 89
11:59:42 [INFO] train episode 496: reward = 92.00, steps = 92
11:59:45 [INFO] train episode 497: reward = 80.00, steps = 80
11:59:49 [INFO] train episode 498: reward = 80.00, steps = 80
11:59:52 [INFO] train episode 499: reward = 74.00, steps = 74
11:59:56 [INFO] train episode 500: reward = 72.00, steps = 72
12:00:00 [INFO] train episode 501: reward = 99.00, steps = 99
12:00:04 [INFO] train episode 502: reward = 83.00, steps = 83
12:00:11 [INFO] train episode 503: reward = 155.00, steps = 155
12:00:16 [INFO] train episode 504: reward = 100.00, steps = 100
12:00:21 [INFO] train episode 505: reward = 107.00, steps = 107
12:00:26 [INFO] train episode 506: reward = 99.00, steps = 99
12:00:29 [INFO] train episode 507: reward = 66.00, steps = 66
12:00:32 [INFO] train episode 508: reward = 81.00, steps = 81
12:00:39 [INFO] train episode 509: reward = 131.00, steps = 131
12:00:42 [INFO] train episode 510: reward = 74.00, steps = 74
12:00:47 [INFO] train episode 511: reward = 111.00, steps = 111
12:00:52 [INFO] train episode 512: reward = 90.00, steps = 90
12:00:56 [INFO] train episode 513: reward = 88.00, steps = 88
12:01:00 [INFO] train episode 514: reward = 92.00, steps = 92
12:01:04 [INFO] train episode 515: reward = 89.00, steps = 89
12:01:07 [INFO] train episode 516: reward = 59.00, steps = 59
12:01:11 [INFO] train episode 517: reward = 82.00, steps = 82
12:01:15 [INFO] train episode 518: reward = 77.00, steps = 77
12:01:19 [INFO] train episode 519: reward = 90.00, steps = 90
12:01:24 [INFO] train episode 520: reward = 116.00, steps = 116
12:01:27 [INFO] train episode 521: reward = 69.00, steps = 69
12:01:33 [INFO] train episode 522: reward = 124.00, steps = 124
12:01:41 [INFO] train episode 523: reward = 169.00, steps = 169
12:01:46 [INFO] train episode 524: reward = 104.00, steps = 104
12:01:49 [INFO] train episode 525: reward = 76.00, steps = 76
12:01:53 [INFO] train episode 526: reward = 75.00, steps = 75
12:01:58 [INFO] train episode 527: reward = 119.00, steps = 119
12:02:03 [INFO] train episode 528: reward = 96.00, steps = 96
12:02:09 [INFO] train episode 529: reward = 135.00, steps = 135
12:02:13 [INFO] train episode 530: reward = 86.00, steps = 86
12:02:19 [INFO] train episode 531: reward = 149.00, steps = 149
12:02:25 [INFO] train episode 532: reward = 131.00, steps = 131
12:02:29 [INFO] train episode 533: reward = 76.00, steps = 76
12:02:38 [INFO] train episode 534: reward = 200.00, steps = 200
12:02:42 [INFO] train episode 535: reward = 89.00, steps = 89
12:02:47 [INFO] train episode 536: reward = 102.00, steps = 102
12:02:51 [INFO] train episode 537: reward = 101.00, steps = 101
12:02:55 [INFO] train episode 538: reward = 79.00, steps = 79
12:02:59 [INFO] train episode 539: reward = 91.00, steps = 91
12:03:03 [INFO] train episode 540: reward = 79.00, steps = 79
12:03:09 [INFO] train episode 541: reward = 129.00, steps = 129
12:03:13 [INFO] train episode 542: reward = 101.00, steps = 101
12:03:19 [INFO] train episode 543: reward = 136.00, steps = 136
12:03:24 [INFO] train episode 544: reward = 97.00, steps = 97
12:03:30 [INFO] train episode 545: reward = 132.00, steps = 132
12:03:35 [INFO] train episode 546: reward = 118.00, steps = 118
12:03:41 [INFO] train episode 547: reward = 136.00, steps = 136
12:03:47 [INFO] train episode 548: reward = 121.00, steps = 121
12:03:50 [INFO] train episode 549: reward = 75.00, steps = 75
12:03:56 [INFO] train episode 550: reward = 130.00, steps = 130
12:04:00 [INFO] train episode 551: reward = 84.00, steps = 84
12:04:05 [INFO] train episode 552: reward = 105.00, steps = 105
12:04:11 [INFO] train episode 553: reward = 147.00, steps = 147
12:04:18 [INFO] train episode 554: reward = 141.00, steps = 141
12:04:22 [INFO] train episode 555: reward = 81.00, steps = 81
12:04:26 [INFO] train episode 556: reward = 95.00, steps = 95
12:04:31 [INFO] train episode 557: reward = 106.00, steps = 106
12:04:38 [INFO] train episode 558: reward = 150.00, steps = 150
12:04:41 [INFO] train episode 559: reward = 81.00, steps = 81
12:04:46 [INFO] train episode 560: reward = 100.00, steps = 100
12:04:52 [INFO] train episode 561: reward = 138.00, steps = 138
12:04:59 [INFO] train episode 562: reward = 136.00, steps = 136
12:05:03 [INFO] train episode 563: reward = 96.00, steps = 96
12:05:06 [INFO] train episode 564: reward = 73.00, steps = 73
12:05:15 [INFO] train episode 565: reward = 200.00, steps = 200
12:05:19 [INFO] train episode 566: reward = 78.00, steps = 78
12:05:27 [INFO] train episode 567: reward = 183.00, steps = 183
12:05:34 [INFO] train episode 568: reward = 158.00, steps = 158
12:05:38 [INFO] train episode 569: reward = 82.00, steps = 82
12:05:45 [INFO] train episode 570: reward = 136.00, steps = 136
12:05:48 [INFO] train episode 571: reward = 69.00, steps = 69
12:05:53 [INFO] train episode 572: reward = 101.00, steps = 101
12:05:59 [INFO] train episode 573: reward = 128.00, steps = 128
12:06:04 [INFO] train episode 574: reward = 114.00, steps = 114
12:06:08 [INFO] train episode 575: reward = 84.00, steps = 84
12:06:10 [INFO] train episode 576: reward = 60.00, steps = 60
12:06:15 [INFO] train episode 577: reward = 99.00, steps = 99
12:06:20 [INFO] train episode 578: reward = 117.00, steps = 117
12:06:25 [INFO] train episode 579: reward = 95.00, steps = 95
12:06:31 [INFO] train episode 580: reward = 143.00, steps = 143
12:06:35 [INFO] train episode 581: reward = 82.00, steps = 82
12:06:42 [INFO] train episode 582: reward = 163.00, steps = 163
12:06:50 [INFO] train episode 583: reward = 176.00, steps = 176
12:06:55 [INFO] train episode 584: reward = 98.00, steps = 98
12:06:58 [INFO] train episode 585: reward = 79.00, steps = 79
12:07:06 [INFO] train episode 586: reward = 166.00, steps = 166
12:07:12 [INFO] train episode 587: reward = 136.00, steps = 136
12:07:16 [INFO] train episode 588: reward = 83.00, steps = 83
12:07:25 [INFO] train episode 589: reward = 200.00, steps = 200
12:07:30 [INFO] train episode 590: reward = 101.00, steps = 101
12:07:36 [INFO] train episode 591: reward = 129.00, steps = 129
12:07:42 [INFO] train episode 592: reward = 134.00, steps = 134
12:07:47 [INFO] train episode 593: reward = 124.00, steps = 124
12:07:53 [INFO] train episode 594: reward = 122.00, steps = 122
12:07:58 [INFO] train episode 595: reward = 106.00, steps = 106
12:08:02 [INFO] train episode 596: reward = 103.00, steps = 103
12:08:07 [INFO] train episode 597: reward = 100.00, steps = 100
12:08:12 [INFO] train episode 598: reward = 104.00, steps = 104
12:08:16 [INFO] train episode 599: reward = 89.00, steps = 89
12:08:21 [INFO] train episode 600: reward = 105.00, steps = 105
12:08:25 [INFO] train episode 601: reward = 96.00, steps = 96
12:08:31 [INFO] train episode 602: reward = 123.00, steps = 123
12:08:35 [INFO] train episode 603: reward = 96.00, steps = 96
12:08:39 [INFO] train episode 604: reward = 96.00, steps = 96
12:08:46 [INFO] train episode 605: reward = 142.00, steps = 142
12:08:52 [INFO] train episode 606: reward = 138.00, steps = 138
12:08:57 [INFO] train episode 607: reward = 101.00, steps = 101
12:09:05 [INFO] train episode 608: reward = 184.00, steps = 184
12:09:09 [INFO] train episode 609: reward = 76.00, steps = 76
12:09:13 [INFO] train episode 610: reward = 93.00, steps = 93
12:09:19 [INFO] train episode 611: reward = 130.00, steps = 130
12:09:25 [INFO] train episode 612: reward = 129.00, steps = 129
12:09:31 [INFO] train episode 613: reward = 133.00, steps = 133
12:09:35 [INFO] train episode 614: reward = 98.00, steps = 98
12:09:39 [INFO] train episode 615: reward = 86.00, steps = 86
12:09:46 [INFO] train episode 616: reward = 146.00, steps = 146
12:09:50 [INFO] train episode 617: reward = 71.00, steps = 71
12:09:54 [INFO] train episode 618: reward = 82.00, steps = 82
12:10:00 [INFO] train episode 619: reward = 142.00, steps = 142
12:10:04 [INFO] train episode 620: reward = 87.00, steps = 87
12:10:12 [INFO] train episode 621: reward = 165.00, steps = 165
12:10:15 [INFO] train episode 622: reward = 86.00, steps = 86
12:10:19 [INFO] train episode 623: reward = 84.00, steps = 84
12:10:25 [INFO] train episode 624: reward = 117.00, steps = 117
12:10:29 [INFO] train episode 625: reward = 88.00, steps = 88
12:10:32 [INFO] train episode 626: reward = 79.00, steps = 79
12:10:38 [INFO] train episode 627: reward = 119.00, steps = 119
12:10:42 [INFO] train episode 628: reward = 88.00, steps = 88
12:10:48 [INFO] train episode 629: reward = 125.00, steps = 125
12:10:51 [INFO] train episode 630: reward = 70.00, steps = 70
12:10:56 [INFO] train episode 631: reward = 113.00, steps = 113
12:11:05 [INFO] train episode 632: reward = 188.00, steps = 188
12:11:10 [INFO] train episode 633: reward = 117.00, steps = 117
12:11:14 [INFO] train episode 634: reward = 94.00, steps = 94
12:11:21 [INFO] train episode 635: reward = 148.00, steps = 148
12:11:26 [INFO] train episode 636: reward = 112.00, steps = 112
12:11:31 [INFO] train episode 637: reward = 117.00, steps = 117
12:11:37 [INFO] train episode 638: reward = 125.00, steps = 125
12:11:41 [INFO] train episode 639: reward = 87.00, steps = 87
12:11:45 [INFO] train episode 640: reward = 95.00, steps = 95
12:11:49 [INFO] train episode 641: reward = 81.00, steps = 81
12:11:54 [INFO] train episode 642: reward = 100.00, steps = 100
12:12:00 [INFO] train episode 643: reward = 143.00, steps = 143
12:12:04 [INFO] train episode 644: reward = 79.00, steps = 79
12:12:11 [INFO] train episode 645: reward = 153.00, steps = 153
12:12:18 [INFO] train episode 646: reward = 166.00, steps = 166
12:12:22 [INFO] train episode 647: reward = 92.00, steps = 92
12:12:28 [INFO] train episode 648: reward = 129.00, steps = 129
12:12:33 [INFO] train episode 649: reward = 112.00, steps = 112
12:12:37 [INFO] train episode 650: reward = 90.00, steps = 90
12:12:41 [INFO] train episode 651: reward = 75.00, steps = 75
12:12:48 [INFO] train episode 652: reward = 143.00, steps = 143
12:12:53 [INFO] train episode 653: reward = 108.00, steps = 108
12:12:58 [INFO] train episode 654: reward = 118.00, steps = 118
12:13:03 [INFO] train episode 655: reward = 101.00, steps = 101
12:13:11 [INFO] train episode 656: reward = 160.00, steps = 160
12:13:17 [INFO] train episode 657: reward = 147.00, steps = 147
12:13:22 [INFO] train episode 658: reward = 96.00, steps = 96
12:13:26 [INFO] train episode 659: reward = 93.00, steps = 93
12:13:33 [INFO] train episode 660: reward = 160.00, steps = 160
12:13:38 [INFO] train episode 661: reward = 95.00, steps = 95
12:13:44 [INFO] train episode 662: reward = 145.00, steps = 145
12:13:53 [INFO] train episode 663: reward = 200.00, steps = 200
12:13:57 [INFO] train episode 664: reward = 92.00, steps = 92
12:14:02 [INFO] train episode 665: reward = 95.00, steps = 95
12:14:05 [INFO] train episode 666: reward = 79.00, steps = 79
12:14:10 [INFO] train episode 667: reward = 106.00, steps = 106
12:14:15 [INFO] train episode 668: reward = 98.00, steps = 98
12:14:22 [INFO] train episode 669: reward = 163.00, steps = 163
12:14:25 [INFO] train episode 670: reward = 71.00, steps = 71
12:14:31 [INFO] train episode 671: reward = 134.00, steps = 134
12:14:37 [INFO] train episode 672: reward = 117.00, steps = 117
12:14:43 [INFO] train episode 673: reward = 138.00, steps = 138
12:14:48 [INFO] train episode 674: reward = 115.00, steps = 115
12:14:55 [INFO] train episode 675: reward = 141.00, steps = 141
12:14:59 [INFO] train episode 676: reward = 84.00, steps = 84
12:15:04 [INFO] train episode 677: reward = 126.00, steps = 126
12:15:10 [INFO] train episode 678: reward = 121.00, steps = 121
12:15:15 [INFO] train episode 679: reward = 105.00, steps = 105
12:15:19 [INFO] train episode 680: reward = 84.00, steps = 84
12:15:24 [INFO] train episode 681: reward = 112.00, steps = 112
12:15:28 [INFO] train episode 682: reward = 101.00, steps = 101
12:15:32 [INFO] train episode 683: reward = 78.00, steps = 78
12:15:38 [INFO] train episode 684: reward = 139.00, steps = 139
12:15:44 [INFO] train episode 685: reward = 135.00, steps = 135
12:15:51 [INFO] train episode 686: reward = 148.00, steps = 148
12:15:58 [INFO] train episode 687: reward = 152.00, steps = 152
12:16:03 [INFO] train episode 688: reward = 119.00, steps = 119
12:16:08 [INFO] train episode 689: reward = 108.00, steps = 108
12:16:15 [INFO] train episode 690: reward = 145.00, steps = 145
12:16:20 [INFO] train episode 691: reward = 113.00, steps = 113
12:16:26 [INFO] train episode 692: reward = 128.00, steps = 128
12:16:31 [INFO] train episode 693: reward = 104.00, steps = 104
12:16:35 [INFO] train episode 694: reward = 93.00, steps = 93
12:16:38 [INFO] train episode 695: reward = 78.00, steps = 78
12:16:42 [INFO] train episode 696: reward = 75.00, steps = 75
12:16:46 [INFO] train episode 697: reward = 93.00, steps = 93
12:16:54 [INFO] train episode 698: reward = 179.00, steps = 179
12:16:59 [INFO] train episode 699: reward = 115.00, steps = 115
12:17:07 [INFO] train episode 700: reward = 167.00, steps = 167
12:17:16 [INFO] train episode 701: reward = 192.00, steps = 192
12:17:20 [INFO] train episode 702: reward = 100.00, steps = 100
12:17:29 [INFO] train episode 703: reward = 200.00, steps = 200
12:17:35 [INFO] train episode 704: reward = 122.00, steps = 122
12:17:44 [INFO] train episode 705: reward = 200.00, steps = 200
12:17:49 [INFO] train episode 706: reward = 118.00, steps = 118
12:17:54 [INFO] train episode 707: reward = 117.00, steps = 117
12:17:59 [INFO] train episode 708: reward = 91.00, steps = 91
12:18:03 [INFO] train episode 709: reward = 89.00, steps = 89
12:18:06 [INFO] train episode 710: reward = 80.00, steps = 80
12:18:15 [INFO] train episode 711: reward = 186.00, steps = 186
12:18:20 [INFO] train episode 712: reward = 125.00, steps = 125
12:18:30 [INFO] train episode 713: reward = 200.00, steps = 200
12:18:33 [INFO] train episode 714: reward = 89.00, steps = 89
12:18:41 [INFO] train episode 715: reward = 170.00, steps = 170
12:18:46 [INFO] train episode 716: reward = 103.00, steps = 103
12:18:55 [INFO] train episode 717: reward = 193.00, steps = 193
12:19:01 [INFO] train episode 718: reward = 132.00, steps = 132
12:19:07 [INFO] train episode 719: reward = 138.00, steps = 138
12:19:12 [INFO] train episode 720: reward = 119.00, steps = 119
12:19:20 [INFO] train episode 721: reward = 173.00, steps = 173
12:19:27 [INFO] train episode 722: reward = 158.00, steps = 158
12:19:33 [INFO] train episode 723: reward = 122.00, steps = 122
12:19:38 [INFO] train episode 724: reward = 100.00, steps = 100
12:19:44 [INFO] train episode 725: reward = 133.00, steps = 133
12:19:49 [INFO] train episode 726: reward = 124.00, steps = 124
12:19:56 [INFO] train episode 727: reward = 148.00, steps = 148
12:20:03 [INFO] train episode 728: reward = 163.00, steps = 163
12:20:08 [INFO] train episode 729: reward = 104.00, steps = 104
12:20:17 [INFO] train episode 730: reward = 193.00, steps = 193
12:20:22 [INFO] train episode 731: reward = 106.00, steps = 106
12:20:27 [INFO] train episode 732: reward = 125.00, steps = 125
12:20:34 [INFO] train episode 733: reward = 154.00, steps = 154
12:20:39 [INFO] train episode 734: reward = 101.00, steps = 101
12:20:47 [INFO] train episode 735: reward = 166.00, steps = 166
12:20:54 [INFO] train episode 736: reward = 155.00, steps = 155
12:20:58 [INFO] train episode 737: reward = 105.00, steps = 105
12:21:03 [INFO] train episode 738: reward = 106.00, steps = 106
12:21:08 [INFO] train episode 739: reward = 113.00, steps = 113
12:21:12 [INFO] train episode 740: reward = 86.00, steps = 86
12:21:19 [INFO] train episode 741: reward = 140.00, steps = 140
12:21:26 [INFO] train episode 742: reward = 155.00, steps = 155
12:21:33 [INFO] train episode 743: reward = 172.00, steps = 172
12:21:39 [INFO] train episode 744: reward = 125.00, steps = 125
12:21:47 [INFO] train episode 745: reward = 180.00, steps = 180
12:21:53 [INFO] train episode 746: reward = 120.00, steps = 120
12:22:00 [INFO] train episode 747: reward = 163.00, steps = 163
12:22:06 [INFO] train episode 748: reward = 138.00, steps = 138
12:22:11 [INFO] train episode 749: reward = 93.00, steps = 93
12:22:18 [INFO] train episode 750: reward = 165.00, steps = 165
12:22:24 [INFO] train episode 751: reward = 135.00, steps = 135
12:22:33 [INFO] train episode 752: reward = 200.00, steps = 200
12:22:41 [INFO] train episode 753: reward = 159.00, steps = 159
12:22:49 [INFO] train episode 754: reward = 185.00, steps = 185
12:22:58 [INFO] train episode 755: reward = 200.00, steps = 200
12:23:06 [INFO] train episode 756: reward = 180.00, steps = 180
12:23:12 [INFO] train episode 757: reward = 125.00, steps = 125
12:23:16 [INFO] train episode 758: reward = 101.00, steps = 101
12:23:21 [INFO] train episode 759: reward = 105.00, steps = 105
12:23:29 [INFO] train episode 760: reward = 168.00, steps = 168
12:23:33 [INFO] train episode 761: reward = 102.00, steps = 102
12:23:39 [INFO] train episode 762: reward = 128.00, steps = 128
12:23:48 [INFO] train episode 763: reward = 200.00, steps = 200
12:23:55 [INFO] train episode 764: reward = 144.00, steps = 144
12:24:02 [INFO] train episode 765: reward = 154.00, steps = 154
12:24:11 [INFO] train episode 766: reward = 195.00, steps = 195
12:24:17 [INFO] train episode 767: reward = 148.00, steps = 148
12:24:24 [INFO] train episode 768: reward = 149.00, steps = 149
12:24:30 [INFO] train episode 769: reward = 128.00, steps = 128
12:24:36 [INFO] train episode 770: reward = 126.00, steps = 126
12:24:42 [INFO] train episode 771: reward = 150.00, steps = 150
12:24:48 [INFO] train episode 772: reward = 127.00, steps = 127
12:24:58 [INFO] train episode 773: reward = 200.00, steps = 200
12:25:05 [INFO] train episode 774: reward = 162.00, steps = 162
12:25:15 [INFO] train episode 775: reward = 200.00, steps = 200
12:25:16 [INFO] train episode 776: reward = 23.00, steps = 23
12:25:22 [INFO] train episode 777: reward = 146.00, steps = 146
12:25:30 [INFO] train episode 778: reward = 177.00, steps = 177
12:25:35 [INFO] train episode 779: reward = 111.00, steps = 111
12:25:43 [INFO] train episode 780: reward = 170.00, steps = 170
12:25:50 [INFO] train episode 781: reward = 155.00, steps = 155
12:25:56 [INFO] train episode 782: reward = 128.00, steps = 128
12:26:03 [INFO] train episode 783: reward = 165.00, steps = 165
12:26:09 [INFO] train episode 784: reward = 129.00, steps = 129
12:26:18 [INFO] train episode 785: reward = 200.00, steps = 200
12:26:26 [INFO] train episode 786: reward = 182.00, steps = 182
12:26:34 [INFO] train episode 787: reward = 168.00, steps = 168
12:26:39 [INFO] train episode 788: reward = 118.00, steps = 118
12:26:47 [INFO] train episode 789: reward = 162.00, steps = 162
12:26:50 [INFO] train episode 790: reward = 63.00, steps = 63
12:26:55 [INFO] train episode 791: reward = 112.00, steps = 112
12:26:57 [INFO] train episode 792: reward = 39.00, steps = 39
12:27:03 [INFO] train episode 793: reward = 143.00, steps = 143
12:27:09 [INFO] train episode 794: reward = 131.00, steps = 131
12:27:14 [INFO] train episode 795: reward = 115.00, steps = 115
12:27:20 [INFO] train episode 796: reward = 134.00, steps = 134
12:27:24 [INFO] train episode 797: reward = 72.00, steps = 72
12:27:30 [INFO] train episode 798: reward = 145.00, steps = 145
12:27:32 [INFO] train episode 799: reward = 43.00, steps = 43
12:27:40 [INFO] train episode 800: reward = 177.00, steps = 177
12:27:43 [INFO] train episode 801: reward = 50.00, steps = 50
12:27:51 [INFO] train episode 802: reward = 196.00, steps = 196
12:28:00 [INFO] train episode 803: reward = 200.00, steps = 200
12:28:10 [INFO] train episode 804: reward = 200.00, steps = 200
12:28:15 [INFO] train episode 805: reward = 110.00, steps = 110
12:28:22 [INFO] train episode 806: reward = 167.00, steps = 167
12:28:27 [INFO] train episode 807: reward = 98.00, steps = 98
12:28:32 [INFO] train episode 808: reward = 116.00, steps = 116
12:28:39 [INFO] train episode 809: reward = 146.00, steps = 146
12:28:47 [INFO] train episode 810: reward = 194.00, steps = 194
12:28:54 [INFO] train episode 811: reward = 141.00, steps = 141
12:29:00 [INFO] train episode 812: reward = 147.00, steps = 147
12:29:07 [INFO] train episode 813: reward = 155.00, steps = 155
12:29:12 [INFO] train episode 814: reward = 107.00, steps = 107
12:29:17 [INFO] train episode 815: reward = 109.00, steps = 109
12:29:25 [INFO] train episode 816: reward = 160.00, steps = 160
12:29:32 [INFO] train episode 817: reward = 165.00, steps = 165
12:29:41 [INFO] train episode 818: reward = 194.00, steps = 194
12:29:47 [INFO] train episode 819: reward = 135.00, steps = 135
12:29:55 [INFO] train episode 820: reward = 183.00, steps = 183
12:30:04 [INFO] train episode 821: reward = 200.00, steps = 200
12:30:10 [INFO] train episode 822: reward = 120.00, steps = 120
12:30:15 [INFO] train episode 823: reward = 115.00, steps = 115
12:30:23 [INFO] train episode 824: reward = 178.00, steps = 178
12:30:30 [INFO] train episode 825: reward = 147.00, steps = 147
12:30:36 [INFO] train episode 826: reward = 133.00, steps = 133
12:30:41 [INFO] train episode 827: reward = 125.00, steps = 125
12:30:46 [INFO] train episode 828: reward = 106.00, steps = 106
12:30:52 [INFO] train episode 829: reward = 136.00, steps = 136
12:30:59 [INFO] train episode 830: reward = 137.00, steps = 137
12:31:06 [INFO] train episode 831: reward = 168.00, steps = 168
12:31:12 [INFO] train episode 832: reward = 123.00, steps = 123
12:31:20 [INFO] train episode 833: reward = 186.00, steps = 186
12:31:26 [INFO] train episode 834: reward = 130.00, steps = 130
12:31:32 [INFO] train episode 835: reward = 126.00, steps = 126
12:31:41 [INFO] train episode 836: reward = 200.00, steps = 200
12:31:50 [INFO] train episode 837: reward = 200.00, steps = 200
12:31:57 [INFO] train episode 838: reward = 151.00, steps = 151
12:32:06 [INFO] train episode 839: reward = 200.00, steps = 200
12:32:12 [INFO] train episode 840: reward = 141.00, steps = 141
12:32:17 [INFO] train episode 841: reward = 108.00, steps = 108
12:32:22 [INFO] train episode 842: reward = 105.00, steps = 105
12:32:28 [INFO] train episode 843: reward = 135.00, steps = 135
12:32:33 [INFO] train episode 844: reward = 109.00, steps = 109
12:32:42 [INFO] train episode 845: reward = 198.00, steps = 198
12:32:47 [INFO] train episode 846: reward = 105.00, steps = 105
12:32:51 [INFO] train episode 847: reward = 103.00, steps = 103
12:32:59 [INFO] train episode 848: reward = 163.00, steps = 163
12:33:04 [INFO] train episode 849: reward = 115.00, steps = 115
12:33:11 [INFO] train episode 850: reward = 154.00, steps = 154
12:33:17 [INFO] train episode 851: reward = 137.00, steps = 137
12:33:22 [INFO] train episode 852: reward = 99.00, steps = 99
12:33:30 [INFO] train episode 853: reward = 171.00, steps = 171
12:33:38 [INFO] train episode 854: reward = 184.00, steps = 184
12:33:43 [INFO] train episode 855: reward = 121.00, steps = 121
12:33:49 [INFO] train episode 856: reward = 132.00, steps = 132
12:33:54 [INFO] train episode 857: reward = 111.00, steps = 111
12:34:01 [INFO] train episode 858: reward = 154.00, steps = 154
12:34:07 [INFO] train episode 859: reward = 131.00, steps = 131
12:34:13 [INFO] train episode 860: reward = 124.00, steps = 124
12:34:19 [INFO] train episode 861: reward = 128.00, steps = 128
12:34:28 [INFO] train episode 862: reward = 200.00, steps = 200
12:34:34 [INFO] train episode 863: reward = 137.00, steps = 137
12:34:42 [INFO] train episode 864: reward = 183.00, steps = 183
12:34:47 [INFO] train episode 865: reward = 101.00, steps = 101
12:34:52 [INFO] train episode 866: reward = 115.00, steps = 115
12:34:58 [INFO] train episode 867: reward = 123.00, steps = 123
12:35:03 [INFO] train episode 868: reward = 110.00, steps = 110
12:35:11 [INFO] train episode 869: reward = 171.00, steps = 171
12:35:16 [INFO] train episode 870: reward = 123.00, steps = 123
12:35:22 [INFO] train episode 871: reward = 113.00, steps = 113
12:35:29 [INFO] train episode 872: reward = 169.00, steps = 169
12:35:36 [INFO] train episode 873: reward = 148.00, steps = 148
12:35:42 [INFO] train episode 874: reward = 132.00, steps = 132
12:35:47 [INFO] train episode 875: reward = 111.00, steps = 111
12:35:54 [INFO] train episode 876: reward = 162.00, steps = 162
12:36:03 [INFO] train episode 877: reward = 200.00, steps = 200
12:36:11 [INFO] train episode 878: reward = 163.00, steps = 163
12:36:18 [INFO] train episode 879: reward = 153.00, steps = 153
12:36:25 [INFO] train episode 880: reward = 162.00, steps = 162
12:36:31 [INFO] train episode 881: reward = 141.00, steps = 141
12:36:37 [INFO] train episode 882: reward = 114.00, steps = 114
12:36:46 [INFO] train episode 883: reward = 200.00, steps = 200
12:36:54 [INFO] train episode 884: reward = 174.00, steps = 174
12:36:59 [INFO] train episode 885: reward = 109.00, steps = 109
12:37:05 [INFO] train episode 886: reward = 131.00, steps = 131
12:37:10 [INFO] train episode 887: reward = 112.00, steps = 112
12:37:20 [INFO] train episode 888: reward = 200.00, steps = 200
12:37:26 [INFO] train episode 889: reward = 152.00, steps = 152
12:37:33 [INFO] train episode 890: reward = 140.00, steps = 140
12:37:39 [INFO] train episode 891: reward = 131.00, steps = 131
12:37:45 [INFO] train episode 892: reward = 141.00, steps = 141
12:37:51 [INFO] train episode 893: reward = 126.00, steps = 126
12:37:55 [INFO] train episode 894: reward = 97.00, steps = 97
12:38:02 [INFO] train episode 895: reward = 152.00, steps = 152
12:38:09 [INFO] train episode 896: reward = 148.00, steps = 148
12:38:14 [INFO] train episode 897: reward = 109.00, steps = 109
12:38:23 [INFO] train episode 898: reward = 200.00, steps = 200
12:38:28 [INFO] train episode 899: reward = 101.00, steps = 101
12:38:34 [INFO] train episode 900: reward = 140.00, steps = 140
12:38:40 [INFO] train episode 901: reward = 134.00, steps = 134
12:38:47 [INFO] train episode 902: reward = 166.00, steps = 166
12:38:56 [INFO] train episode 903: reward = 200.00, steps = 200
12:39:03 [INFO] train episode 904: reward = 152.00, steps = 152
12:39:10 [INFO] train episode 905: reward = 149.00, steps = 149
12:39:15 [INFO] train episode 906: reward = 110.00, steps = 110
12:39:20 [INFO] train episode 907: reward = 111.00, steps = 111
12:39:27 [INFO] train episode 908: reward = 151.00, steps = 151
12:39:34 [INFO] train episode 909: reward = 149.00, steps = 149
12:39:41 [INFO] train episode 910: reward = 151.00, steps = 151
12:39:46 [INFO] train episode 911: reward = 110.00, steps = 110
12:39:53 [INFO] train episode 912: reward = 165.00, steps = 165
12:40:01 [INFO] train episode 913: reward = 165.00, steps = 165
12:40:09 [INFO] train episode 914: reward = 183.00, steps = 183
12:40:16 [INFO] train episode 915: reward = 147.00, steps = 147
12:40:23 [INFO] train episode 916: reward = 164.00, steps = 164
12:40:30 [INFO] train episode 917: reward = 144.00, steps = 144
12:40:39 [INFO] train episode 918: reward = 197.00, steps = 197
12:40:45 [INFO] train episode 919: reward = 118.00, steps = 118
12:40:52 [INFO] train episode 920: reward = 165.00, steps = 165
12:40:59 [INFO] train episode 921: reward = 140.00, steps = 140
12:41:04 [INFO] train episode 922: reward = 116.00, steps = 116
12:41:09 [INFO] train episode 923: reward = 117.00, steps = 117
12:41:15 [INFO] train episode 924: reward = 132.00, steps = 132
12:41:20 [INFO] train episode 925: reward = 94.00, steps = 94
12:41:25 [INFO] train episode 926: reward = 122.00, steps = 122
12:41:32 [INFO] train episode 927: reward = 150.00, steps = 150
12:41:40 [INFO] train episode 928: reward = 172.00, steps = 172
12:41:46 [INFO] train episode 929: reward = 126.00, steps = 126
12:41:51 [INFO] train episode 930: reward = 130.00, steps = 130
12:41:58 [INFO] train episode 931: reward = 140.00, steps = 140
12:42:03 [INFO] train episode 932: reward = 107.00, steps = 107
12:42:12 [INFO] train episode 933: reward = 200.00, steps = 200
12:42:18 [INFO] train episode 934: reward = 129.00, steps = 129
12:42:27 [INFO] train episode 935: reward = 200.00, steps = 200
12:42:33 [INFO] train episode 936: reward = 148.00, steps = 148
12:42:41 [INFO] train episode 937: reward = 164.00, steps = 164
12:42:47 [INFO] train episode 938: reward = 132.00, steps = 132
12:42:52 [INFO] train episode 939: reward = 108.00, steps = 108
12:42:59 [INFO] train episode 940: reward = 149.00, steps = 149
12:43:04 [INFO] train episode 941: reward = 121.00, steps = 121
12:43:11 [INFO] train episode 942: reward = 148.00, steps = 148
12:43:17 [INFO] train episode 943: reward = 138.00, steps = 138
12:43:26 [INFO] train episode 944: reward = 200.00, steps = 200
12:43:31 [INFO] train episode 945: reward = 112.00, steps = 112
12:43:37 [INFO] train episode 946: reward = 123.00, steps = 123
12:43:46 [INFO] train episode 947: reward = 200.00, steps = 200
12:43:52 [INFO] train episode 948: reward = 140.00, steps = 140
12:44:01 [INFO] train episode 949: reward = 185.00, steps = 185
12:44:06 [INFO] train episode 950: reward = 112.00, steps = 112
12:44:11 [INFO] train episode 951: reward = 127.00, steps = 127
12:44:18 [INFO] train episode 952: reward = 154.00, steps = 154
12:44:23 [INFO] train episode 953: reward = 100.00, steps = 100
12:44:30 [INFO] train episode 954: reward = 154.00, steps = 154
12:44:38 [INFO] train episode 955: reward = 178.00, steps = 178
12:44:46 [INFO] train episode 956: reward = 163.00, steps = 163
12:44:51 [INFO] train episode 957: reward = 104.00, steps = 104
12:45:00 [INFO] train episode 958: reward = 200.00, steps = 200
12:45:09 [INFO] train episode 959: reward = 196.00, steps = 196
12:45:17 [INFO] train episode 960: reward = 177.00, steps = 177
12:45:22 [INFO] train episode 961: reward = 119.00, steps = 119
12:45:29 [INFO] train episode 962: reward = 148.00, steps = 148
12:45:35 [INFO] train episode 963: reward = 122.00, steps = 122
12:45:40 [INFO] train episode 964: reward = 113.00, steps = 113
12:45:48 [INFO] train episode 965: reward = 175.00, steps = 175
12:45:54 [INFO] train episode 966: reward = 147.00, steps = 147
12:46:01 [INFO] train episode 967: reward = 147.00, steps = 147
12:46:07 [INFO] train episode 968: reward = 130.00, steps = 130
12:46:14 [INFO] train episode 969: reward = 162.00, steps = 162
12:46:22 [INFO] train episode 970: reward = 179.00, steps = 179
12:46:28 [INFO] train episode 971: reward = 130.00, steps = 130
12:46:37 [INFO] train episode 972: reward = 199.00, steps = 199
12:46:46 [INFO] train episode 973: reward = 200.00, steps = 200
12:46:51 [INFO] train episode 974: reward = 109.00, steps = 109
12:47:00 [INFO] train episode 975: reward = 193.00, steps = 193
12:47:04 [INFO] train episode 976: reward = 97.00, steps = 97
12:47:12 [INFO] train episode 977: reward = 165.00, steps = 165
12:47:17 [INFO] train episode 978: reward = 115.00, steps = 115
12:47:24 [INFO] train episode 979: reward = 159.00, steps = 159
12:47:31 [INFO] train episode 980: reward = 139.00, steps = 139
12:47:38 [INFO] train episode 981: reward = 173.00, steps = 173
12:47:47 [INFO] train episode 982: reward = 195.00, steps = 195
12:47:53 [INFO] train episode 983: reward = 129.00, steps = 129
12:48:01 [INFO] train episode 984: reward = 171.00, steps = 171
12:48:08 [INFO] train episode 985: reward = 157.00, steps = 157
12:48:13 [INFO] train episode 986: reward = 110.00, steps = 110
12:48:18 [INFO] train episode 987: reward = 113.00, steps = 113
12:48:22 [INFO] train episode 988: reward = 90.00, steps = 90
12:48:28 [INFO] train episode 989: reward = 125.00, steps = 125
12:48:33 [INFO] train episode 990: reward = 104.00, steps = 104
12:48:38 [INFO] train episode 991: reward = 119.00, steps = 119
12:48:44 [INFO] train episode 992: reward = 127.00, steps = 127
12:48:51 [INFO] train episode 993: reward = 157.00, steps = 157
12:48:57 [INFO] train episode 994: reward = 116.00, steps = 116
12:49:01 [INFO] train episode 995: reward = 104.00, steps = 104
12:49:05 [INFO] train episode 996: reward = 88.00, steps = 88
12:49:13 [INFO] train episode 997: reward = 179.00, steps = 179
12:49:21 [INFO] train episode 998: reward = 159.00, steps = 159
12:49:26 [INFO] train episode 999: reward = 125.00, steps = 125
12:49:31 [INFO] train episode 1000: reward = 98.00, steps = 98
12:49:35 [INFO] train episode 1001: reward = 96.00, steps = 96
12:49:41 [INFO] train episode 1002: reward = 123.00, steps = 123
12:49:45 [INFO] train episode 1003: reward = 93.00, steps = 93
12:49:50 [INFO] train episode 1004: reward = 111.00, steps = 111
12:49:57 [INFO] train episode 1005: reward = 151.00, steps = 151
12:50:02 [INFO] train episode 1006: reward = 111.00, steps = 111
12:50:10 [INFO] train episode 1007: reward = 170.00, steps = 170
12:50:14 [INFO] train episode 1008: reward = 95.00, steps = 95
12:50:19 [INFO] train episode 1009: reward = 105.00, steps = 105
12:50:25 [INFO] train episode 1010: reward = 130.00, steps = 130
12:50:30 [INFO] train episode 1011: reward = 101.00, steps = 101
12:50:36 [INFO] train episode 1012: reward = 130.00, steps = 130
12:50:39 [INFO] train episode 1013: reward = 83.00, steps = 83
12:50:45 [INFO] train episode 1014: reward = 127.00, steps = 127
12:50:53 [INFO] train episode 1015: reward = 163.00, steps = 163
12:51:00 [INFO] train episode 1016: reward = 166.00, steps = 166
12:51:06 [INFO] train episode 1017: reward = 130.00, steps = 130
12:51:10 [INFO] train episode 1018: reward = 95.00, steps = 95
12:51:16 [INFO] train episode 1019: reward = 134.00, steps = 134
12:51:23 [INFO] train episode 1020: reward = 139.00, steps = 139
12:51:27 [INFO] train episode 1021: reward = 96.00, steps = 96
12:51:34 [INFO] train episode 1022: reward = 155.00, steps = 155
12:51:39 [INFO] train episode 1023: reward = 107.00, steps = 107
12:51:44 [INFO] train episode 1024: reward = 107.00, steps = 107
12:51:49 [INFO] train episode 1025: reward = 105.00, steps = 105
12:51:56 [INFO] train episode 1026: reward = 147.00, steps = 147
12:52:01 [INFO] train episode 1027: reward = 125.00, steps = 125
12:52:06 [INFO] train episode 1028: reward = 108.00, steps = 108
12:52:11 [INFO] train episode 1029: reward = 108.00, steps = 108
12:52:17 [INFO] train episode 1030: reward = 129.00, steps = 129
12:52:22 [INFO] train episode 1031: reward = 113.00, steps = 113
12:52:26 [INFO] train episode 1032: reward = 80.00, steps = 80
12:52:31 [INFO] train episode 1033: reward = 106.00, steps = 106
12:52:35 [INFO] train episode 1034: reward = 91.00, steps = 91
12:52:40 [INFO] train episode 1035: reward = 106.00, steps = 106
12:52:45 [INFO] train episode 1036: reward = 121.00, steps = 121
12:52:49 [INFO] train episode 1037: reward = 91.00, steps = 91
12:52:55 [INFO] train episode 1038: reward = 114.00, steps = 114
12:52:59 [INFO] train episode 1039: reward = 103.00, steps = 103
12:53:05 [INFO] train episode 1040: reward = 111.00, steps = 111
12:53:11 [INFO] train episode 1041: reward = 154.00, steps = 154
12:53:16 [INFO] train episode 1042: reward = 93.00, steps = 93
12:53:22 [INFO] train episode 1043: reward = 129.00, steps = 129
12:53:31 [INFO] train episode 1044: reward = 200.00, steps = 200
12:53:37 [INFO] train episode 1045: reward = 128.00, steps = 128
12:53:42 [INFO] train episode 1046: reward = 121.00, steps = 121
12:53:51 [INFO] train episode 1047: reward = 200.00, steps = 200
12:53:57 [INFO] train episode 1048: reward = 130.00, steps = 130
12:54:01 [INFO] train episode 1049: reward = 93.00, steps = 93
12:54:06 [INFO] train episode 1050: reward = 94.00, steps = 94
12:54:11 [INFO] train episode 1051: reward = 118.00, steps = 118
12:54:19 [INFO] train episode 1052: reward = 171.00, steps = 171
12:54:23 [INFO] train episode 1053: reward = 101.00, steps = 101
12:54:28 [INFO] train episode 1054: reward = 95.00, steps = 95
12:54:36 [INFO] train episode 1055: reward = 180.00, steps = 180
12:54:40 [INFO] train episode 1056: reward = 98.00, steps = 98
12:54:45 [INFO] train episode 1057: reward = 109.00, steps = 109
12:54:51 [INFO] train episode 1058: reward = 122.00, steps = 122
12:54:56 [INFO] train episode 1059: reward = 100.00, steps = 100
12:55:02 [INFO] train episode 1060: reward = 142.00, steps = 142
12:55:06 [INFO] train episode 1061: reward = 96.00, steps = 96
12:55:12 [INFO] train episode 1062: reward = 130.00, steps = 130
12:55:18 [INFO] train episode 1063: reward = 126.00, steps = 126
12:55:28 [INFO] train episode 1064: reward = 200.00, steps = 200
12:55:34 [INFO] train episode 1065: reward = 132.00, steps = 132
12:55:38 [INFO] train episode 1066: reward = 98.00, steps = 98
12:55:43 [INFO] train episode 1067: reward = 90.00, steps = 90
12:55:48 [INFO] train episode 1068: reward = 128.00, steps = 128
12:55:57 [INFO] train episode 1069: reward = 182.00, steps = 182
12:56:01 [INFO] train episode 1070: reward = 105.00, steps = 105
12:56:07 [INFO] train episode 1071: reward = 131.00, steps = 131
12:56:12 [INFO] train episode 1072: reward = 100.00, steps = 100
12:56:20 [INFO] train episode 1073: reward = 186.00, steps = 186
12:56:29 [INFO] train episode 1074: reward = 193.00, steps = 193
12:56:34 [INFO] train episode 1075: reward = 100.00, steps = 100
12:56:43 [INFO] train episode 1076: reward = 200.00, steps = 200
12:56:49 [INFO] train episode 1077: reward = 146.00, steps = 146
12:56:58 [INFO] train episode 1078: reward = 200.00, steps = 200
12:57:04 [INFO] train episode 1079: reward = 116.00, steps = 116
12:57:10 [INFO] train episode 1080: reward = 131.00, steps = 131
12:57:16 [INFO] train episode 1081: reward = 144.00, steps = 144
12:57:22 [INFO] train episode 1082: reward = 118.00, steps = 118
12:57:28 [INFO] train episode 1083: reward = 130.00, steps = 130
12:57:37 [INFO] train episode 1084: reward = 200.00, steps = 200
12:57:43 [INFO] train episode 1085: reward = 154.00, steps = 154
12:57:50 [INFO] train episode 1086: reward = 155.00, steps = 155
12:57:59 [INFO] train episode 1087: reward = 179.00, steps = 179
12:58:04 [INFO] train episode 1088: reward = 119.00, steps = 119
12:58:08 [INFO] train episode 1089: reward = 93.00, steps = 93
12:58:13 [INFO] train episode 1090: reward = 111.00, steps = 111
12:58:18 [INFO] train episode 1091: reward = 101.00, steps = 101
12:58:23 [INFO] train episode 1092: reward = 97.00, steps = 97
12:58:27 [INFO] train episode 1093: reward = 102.00, steps = 102
12:58:33 [INFO] train episode 1094: reward = 133.00, steps = 133
12:58:37 [INFO] train episode 1095: reward = 88.00, steps = 88
12:58:43 [INFO] train episode 1096: reward = 126.00, steps = 126
12:58:50 [INFO] train episode 1097: reward = 154.00, steps = 154
12:58:56 [INFO] train episode 1098: reward = 142.00, steps = 142
12:59:02 [INFO] train episode 1099: reward = 115.00, steps = 115
12:59:06 [INFO] train episode 1100: reward = 95.00, steps = 95
12:59:12 [INFO] train episode 1101: reward = 123.00, steps = 123
12:59:17 [INFO] train episode 1102: reward = 119.00, steps = 119
12:59:22 [INFO] train episode 1103: reward = 119.00, steps = 119
12:59:29 [INFO] train episode 1104: reward = 140.00, steps = 140
12:59:34 [INFO] train episode 1105: reward = 119.00, steps = 119
12:59:39 [INFO] train episode 1106: reward = 112.00, steps = 112
12:59:47 [INFO] train episode 1107: reward = 171.00, steps = 171
12:59:53 [INFO] train episode 1108: reward = 141.00, steps = 141
12:59:58 [INFO] train episode 1109: reward = 109.00, steps = 109
13:00:04 [INFO] train episode 1110: reward = 130.00, steps = 130
13:00:08 [INFO] train episode 1111: reward = 84.00, steps = 84
13:00:14 [INFO] train episode 1112: reward = 121.00, steps = 121
13:00:20 [INFO] train episode 1113: reward = 143.00, steps = 143
13:00:25 [INFO] train episode 1114: reward = 114.00, steps = 114
13:00:30 [INFO] train episode 1115: reward = 98.00, steps = 98
13:00:39 [INFO] train episode 1116: reward = 194.00, steps = 194
13:00:43 [INFO] train episode 1117: reward = 99.00, steps = 99
13:00:49 [INFO] train episode 1118: reward = 134.00, steps = 134
13:00:55 [INFO] train episode 1119: reward = 130.00, steps = 130
13:01:00 [INFO] train episode 1120: reward = 95.00, steps = 95
13:01:04 [INFO] train episode 1121: reward = 101.00, steps = 101
13:01:11 [INFO] train episode 1122: reward = 146.00, steps = 146
13:01:19 [INFO] train episode 1123: reward = 183.00, steps = 183
13:01:27 [INFO] train episode 1124: reward = 181.00, steps = 181
13:01:33 [INFO] train episode 1125: reward = 128.00, steps = 128
13:01:39 [INFO] train episode 1126: reward = 138.00, steps = 138
13:01:45 [INFO] train episode 1127: reward = 111.00, steps = 111
13:01:53 [INFO] train episode 1128: reward = 189.00, steps = 189
13:01:59 [INFO] train episode 1129: reward = 124.00, steps = 124
13:02:03 [INFO] train episode 1130: reward = 94.00, steps = 94
13:02:09 [INFO] train episode 1131: reward = 122.00, steps = 122
13:02:18 [INFO] train episode 1132: reward = 200.00, steps = 200
13:02:22 [INFO] train episode 1133: reward = 102.00, steps = 102
13:02:31 [INFO] train episode 1134: reward = 200.00, steps = 200
13:02:38 [INFO] train episode 1135: reward = 139.00, steps = 139
13:02:43 [INFO] train episode 1136: reward = 122.00, steps = 122
13:02:48 [INFO] train episode 1137: reward = 106.00, steps = 106
13:02:53 [INFO] train episode 1138: reward = 101.00, steps = 101
13:02:58 [INFO] train episode 1139: reward = 110.00, steps = 110
13:03:06 [INFO] train episode 1140: reward = 168.00, steps = 168
13:03:15 [INFO] train episode 1141: reward = 200.00, steps = 200
13:03:20 [INFO] train episode 1142: reward = 123.00, steps = 123
13:03:26 [INFO] train episode 1143: reward = 138.00, steps = 138
13:03:32 [INFO] train episode 1144: reward = 116.00, steps = 116
13:03:39 [INFO] train episode 1145: reward = 162.00, steps = 162
13:03:45 [INFO] train episode 1146: reward = 137.00, steps = 137
13:03:53 [INFO] train episode 1147: reward = 173.00, steps = 173
13:03:58 [INFO] train episode 1148: reward = 116.00, steps = 116
13:04:03 [INFO] train episode 1149: reward = 106.00, steps = 106
13:04:08 [INFO] train episode 1150: reward = 110.00, steps = 110
13:04:13 [INFO] train episode 1151: reward = 110.00, steps = 110
13:04:21 [INFO] train episode 1152: reward = 158.00, steps = 158
13:04:28 [INFO] train episode 1153: reward = 164.00, steps = 164
13:04:36 [INFO] train episode 1154: reward = 175.00, steps = 175
13:04:40 [INFO] train episode 1155: reward = 88.00, steps = 88
13:04:45 [INFO] train episode 1156: reward = 111.00, steps = 111
13:04:54 [INFO] train episode 1157: reward = 182.00, steps = 182
13:04:59 [INFO] train episode 1158: reward = 120.00, steps = 120
13:05:03 [INFO] train episode 1159: reward = 89.00, steps = 89
13:05:08 [INFO] train episode 1160: reward = 118.00, steps = 118
13:05:15 [INFO] train episode 1161: reward = 155.00, steps = 155
13:05:19 [INFO] train episode 1162: reward = 82.00, steps = 82
13:05:24 [INFO] train episode 1163: reward = 111.00, steps = 111
13:05:32 [INFO] train episode 1164: reward = 173.00, steps = 173
13:05:37 [INFO] train episode 1165: reward = 101.00, steps = 101
13:05:43 [INFO] train episode 1166: reward = 128.00, steps = 128
13:05:49 [INFO] train episode 1167: reward = 133.00, steps = 133
13:05:55 [INFO] train episode 1168: reward = 136.00, steps = 136
13:06:01 [INFO] train episode 1169: reward = 137.00, steps = 137
13:06:05 [INFO] train episode 1170: reward = 94.00, steps = 94
13:06:13 [INFO] train episode 1171: reward = 163.00, steps = 163
13:06:17 [INFO] train episode 1172: reward = 102.00, steps = 102
13:06:25 [INFO] train episode 1173: reward = 173.00, steps = 173
13:06:34 [INFO] train episode 1174: reward = 200.00, steps = 200
13:06:42 [INFO] train episode 1175: reward = 180.00, steps = 180
13:06:48 [INFO] train episode 1176: reward = 125.00, steps = 125
13:06:53 [INFO] train episode 1177: reward = 104.00, steps = 104
13:06:59 [INFO] train episode 1178: reward = 123.00, steps = 123
13:07:03 [INFO] train episode 1179: reward = 102.00, steps = 102
13:07:08 [INFO] train episode 1180: reward = 105.00, steps = 105
13:07:12 [INFO] train episode 1181: reward = 89.00, steps = 89
13:07:17 [INFO] train episode 1182: reward = 101.00, steps = 101
13:07:22 [INFO] train episode 1183: reward = 106.00, steps = 106
13:07:26 [INFO] train episode 1184: reward = 98.00, steps = 98
13:07:32 [INFO] train episode 1185: reward = 119.00, steps = 119
13:07:37 [INFO] train episode 1186: reward = 106.00, steps = 106
13:07:42 [INFO] train episode 1187: reward = 114.00, steps = 114
13:07:47 [INFO] train episode 1188: reward = 95.00, steps = 95
13:07:53 [INFO] train episode 1189: reward = 127.00, steps = 127
13:07:57 [INFO] train episode 1190: reward = 101.00, steps = 101
13:08:02 [INFO] train episode 1191: reward = 106.00, steps = 106
13:08:08 [INFO] train episode 1192: reward = 120.00, steps = 120
13:08:16 [INFO] train episode 1193: reward = 182.00, steps = 182
13:08:22 [INFO] train episode 1194: reward = 125.00, steps = 125
13:08:27 [INFO] train episode 1195: reward = 127.00, steps = 127
13:08:34 [INFO] train episode 1196: reward = 125.00, steps = 125
13:08:40 [INFO] train episode 1197: reward = 128.00, steps = 128
13:08:44 [INFO] train episode 1198: reward = 93.00, steps = 93
13:08:49 [INFO] train episode 1199: reward = 116.00, steps = 116
13:08:57 [INFO] train episode 1200: reward = 155.00, steps = 155
13:09:02 [INFO] train episode 1201: reward = 114.00, steps = 114
13:09:10 [INFO] train episode 1202: reward = 153.00, steps = 153
13:09:17 [INFO] train episode 1203: reward = 135.00, steps = 135
13:09:27 [INFO] train episode 1204: reward = 200.00, steps = 200
13:09:32 [INFO] train episode 1205: reward = 93.00, steps = 93
13:09:37 [INFO] train episode 1206: reward = 96.00, steps = 96
13:09:43 [INFO] train episode 1207: reward = 128.00, steps = 128
13:09:48 [INFO] train episode 1208: reward = 97.00, steps = 97
13:09:55 [INFO] train episode 1209: reward = 133.00, steps = 133
13:10:03 [INFO] train episode 1210: reward = 166.00, steps = 166
13:10:08 [INFO] train episode 1211: reward = 92.00, steps = 92
13:10:14 [INFO] train episode 1212: reward = 113.00, steps = 113
13:10:20 [INFO] train episode 1213: reward = 107.00, steps = 107
13:10:27 [INFO] train episode 1214: reward = 126.00, steps = 126
13:10:34 [INFO] train episode 1215: reward = 136.00, steps = 136
13:10:41 [INFO] train episode 1216: reward = 127.00, steps = 127
13:10:49 [INFO] train episode 1217: reward = 162.00, steps = 162
13:10:54 [INFO] train episode 1218: reward = 93.00, steps = 93
13:10:59 [INFO] train episode 1219: reward = 99.00, steps = 99
13:11:05 [INFO] train episode 1220: reward = 97.00, steps = 97
13:11:13 [INFO] train episode 1221: reward = 124.00, steps = 124
13:11:22 [INFO] train episode 1222: reward = 153.00, steps = 153
13:11:33 [INFO] train episode 1223: reward = 200.00, steps = 200
13:11:39 [INFO] train episode 1224: reward = 109.00, steps = 109
13:11:46 [INFO] train episode 1225: reward = 129.00, steps = 129
13:11:56 [INFO] train episode 1226: reward = 133.00, steps = 133
13:12:03 [INFO] train episode 1227: reward = 116.00, steps = 116
13:12:11 [INFO] train episode 1228: reward = 136.00, steps = 136
13:12:18 [INFO] train episode 1229: reward = 122.00, steps = 122
13:12:27 [INFO] train episode 1230: reward = 166.00, steps = 166
13:12:34 [INFO] train episode 1231: reward = 128.00, steps = 128
13:12:40 [INFO] train episode 1232: reward = 103.00, steps = 103
13:12:46 [INFO] train episode 1233: reward = 103.00, steps = 103
13:12:53 [INFO] train episode 1234: reward = 124.00, steps = 124
13:13:02 [INFO] train episode 1235: reward = 147.00, steps = 147
13:13:10 [INFO] train episode 1236: reward = 141.00, steps = 141
13:13:17 [INFO] train episode 1237: reward = 128.00, steps = 128
13:13:25 [INFO] train episode 1238: reward = 131.00, steps = 131
13:13:31 [INFO] train episode 1239: reward = 107.00, steps = 107
13:13:38 [INFO] train episode 1240: reward = 125.00, steps = 125
13:13:47 [INFO] train episode 1241: reward = 160.00, steps = 160
13:13:54 [INFO] train episode 1242: reward = 119.00, steps = 119
13:14:00 [INFO] train episode 1243: reward = 107.00, steps = 107
13:14:06 [INFO] train episode 1244: reward = 97.00, steps = 97
13:14:12 [INFO] train episode 1245: reward = 112.00, steps = 112
13:14:18 [INFO] train episode 1246: reward = 107.00, steps = 107
13:14:27 [INFO] train episode 1247: reward = 151.00, steps = 151
13:14:34 [INFO] train episode 1248: reward = 124.00, steps = 124
13:14:44 [INFO] train episode 1249: reward = 172.00, steps = 172
13:14:51 [INFO] train episode 1250: reward = 131.00, steps = 131
13:14:58 [INFO] train episode 1251: reward = 115.00, steps = 115
13:15:05 [INFO] train episode 1252: reward = 127.00, steps = 127
13:15:11 [INFO] train episode 1253: reward = 119.00, steps = 119
13:15:17 [INFO] train episode 1254: reward = 86.00, steps = 86
13:15:23 [INFO] train episode 1255: reward = 120.00, steps = 120
13:15:30 [INFO] train episode 1256: reward = 119.00, steps = 119
13:15:39 [INFO] train episode 1257: reward = 146.00, steps = 146
13:15:45 [INFO] train episode 1258: reward = 104.00, steps = 104
13:15:53 [INFO] train episode 1259: reward = 146.00, steps = 146
13:16:01 [INFO] train episode 1260: reward = 139.00, steps = 139
13:16:10 [INFO] train episode 1261: reward = 167.00, steps = 167
13:16:17 [INFO] train episode 1262: reward = 123.00, steps = 123
13:16:24 [INFO] train episode 1263: reward = 127.00, steps = 127
13:16:35 [INFO] train episode 1264: reward = 200.00, steps = 200
13:16:47 [INFO] train episode 1265: reward = 197.00, steps = 197
13:16:53 [INFO] train episode 1266: reward = 111.00, steps = 111
13:17:00 [INFO] train episode 1267: reward = 122.00, steps = 122
13:17:06 [INFO] train episode 1268: reward = 109.00, steps = 109
13:17:17 [INFO] train episode 1269: reward = 183.00, steps = 183
13:17:23 [INFO] train episode 1270: reward = 111.00, steps = 111
13:17:30 [INFO] train episode 1271: reward = 135.00, steps = 135
13:17:36 [INFO] train episode 1272: reward = 102.00, steps = 102
13:17:41 [INFO] train episode 1273: reward = 91.00, steps = 91
13:17:47 [INFO] train episode 1274: reward = 108.00, steps = 108
13:17:52 [INFO] train episode 1275: reward = 80.00, steps = 80
13:17:59 [INFO] train episode 1276: reward = 127.00, steps = 127
13:18:07 [INFO] train episode 1277: reward = 132.00, steps = 132
13:18:16 [INFO] train episode 1278: reward = 156.00, steps = 156
13:18:22 [INFO] train episode 1279: reward = 110.00, steps = 110
13:18:28 [INFO] train episode 1280: reward = 107.00, steps = 107
13:18:34 [INFO] train episode 1281: reward = 104.00, steps = 104
13:18:39 [INFO] train episode 1282: reward = 94.00, steps = 94
13:18:46 [INFO] train episode 1283: reward = 123.00, steps = 123
13:18:53 [INFO] train episode 1284: reward = 131.00, steps = 131
13:19:00 [INFO] train episode 1285: reward = 113.00, steps = 113
13:19:05 [INFO] train episode 1286: reward = 106.00, steps = 106
13:19:16 [INFO] train episode 1287: reward = 160.00, steps = 160
13:19:24 [INFO] train episode 1288: reward = 104.00, steps = 104
13:19:29 [INFO] train episode 1289: reward = 86.00, steps = 86
13:19:36 [INFO] train episode 1290: reward = 96.00, steps = 96
13:19:43 [INFO] train episode 1291: reward = 120.00, steps = 120
13:19:49 [INFO] train episode 1292: reward = 111.00, steps = 111
13:19:57 [INFO] train episode 1293: reward = 129.00, steps = 129
13:20:05 [INFO] train episode 1294: reward = 131.00, steps = 131
13:20:13 [INFO] train episode 1295: reward = 143.00, steps = 143
13:20:21 [INFO] train episode 1296: reward = 126.00, steps = 126
13:20:31 [INFO] train episode 1297: reward = 167.00, steps = 167
13:20:39 [INFO] train episode 1298: reward = 135.00, steps = 135
13:20:46 [INFO] train episode 1299: reward = 117.00, steps = 117
13:20:53 [INFO] train episode 1300: reward = 126.00, steps = 126
13:21:00 [INFO] train episode 1301: reward = 106.00, steps = 106
13:21:06 [INFO] train episode 1302: reward = 107.00, steps = 107
13:21:14 [INFO] train episode 1303: reward = 141.00, steps = 141
13:21:21 [INFO] train episode 1304: reward = 123.00, steps = 123
13:21:28 [INFO] train episode 1305: reward = 117.00, steps = 117
13:21:36 [INFO] train episode 1306: reward = 136.00, steps = 136
13:21:46 [INFO] train episode 1307: reward = 172.00, steps = 172
13:21:56 [INFO] train episode 1308: reward = 172.00, steps = 172
13:22:03 [INFO] train episode 1309: reward = 113.00, steps = 113
13:22:11 [INFO] train episode 1310: reward = 142.00, steps = 142
13:22:18 [INFO] train episode 1311: reward = 132.00, steps = 132
13:22:23 [INFO] train episode 1312: reward = 92.00, steps = 92
13:22:31 [INFO] train episode 1313: reward = 133.00, steps = 133
13:22:41 [INFO] train episode 1314: reward = 148.00, steps = 148
13:22:48 [INFO] train episode 1315: reward = 120.00, steps = 120
13:22:58 [INFO] train episode 1316: reward = 142.00, steps = 142
13:23:07 [INFO] train episode 1317: reward = 147.00, steps = 147
13:23:14 [INFO] train episode 1318: reward = 133.00, steps = 133
13:23:27 [INFO] train episode 1319: reward = 200.00, steps = 200
13:23:39 [INFO] train episode 1320: reward = 187.00, steps = 187
13:23:49 [INFO] train episode 1321: reward = 187.00, steps = 187
13:23:58 [INFO] train episode 1322: reward = 163.00, steps = 163
13:24:08 [INFO] train episode 1323: reward = 187.00, steps = 187
13:24:18 [INFO] train episode 1324: reward = 182.00, steps = 182
13:24:28 [INFO] train episode 1325: reward = 195.00, steps = 195
13:24:37 [INFO] train episode 1326: reward = 151.00, steps = 151
13:24:43 [INFO] train episode 1327: reward = 117.00, steps = 117
13:24:53 [INFO] train episode 1328: reward = 181.00, steps = 181
13:25:01 [INFO] train episode 1329: reward = 147.00, steps = 147
13:25:08 [INFO] train episode 1330: reward = 122.00, steps = 122
13:25:19 [INFO] train episode 1331: reward = 200.00, steps = 200
13:25:30 [INFO] train episode 1332: reward = 200.00, steps = 200
13:25:38 [INFO] train episode 1333: reward = 154.00, steps = 154
13:25:46 [INFO] train episode 1334: reward = 144.00, steps = 144
13:25:53 [INFO] train episode 1335: reward = 132.00, steps = 132
13:26:04 [INFO] train episode 1336: reward = 200.00, steps = 200
13:26:11 [INFO] train episode 1337: reward = 137.00, steps = 137
13:26:21 [INFO] train episode 1338: reward = 166.00, steps = 166
13:26:31 [INFO] train episode 1339: reward = 200.00, steps = 200
13:26:38 [INFO] train episode 1340: reward = 125.00, steps = 125
13:26:45 [INFO] train episode 1341: reward = 131.00, steps = 131
13:26:53 [INFO] train episode 1342: reward = 153.00, steps = 153
13:27:02 [INFO] train episode 1343: reward = 164.00, steps = 164
13:27:08 [INFO] train episode 1344: reward = 107.00, steps = 107
13:27:16 [INFO] train episode 1345: reward = 143.00, steps = 143
13:27:25 [INFO] train episode 1346: reward = 160.00, steps = 160
13:27:32 [INFO] train episode 1347: reward = 127.00, steps = 127
13:27:41 [INFO] train episode 1348: reward = 165.00, steps = 165
13:27:50 [INFO] train episode 1349: reward = 173.00, steps = 173
13:28:00 [INFO] train episode 1350: reward = 173.00, steps = 173
13:28:10 [INFO] train episode 1351: reward = 200.00, steps = 200
13:28:17 [INFO] train episode 1352: reward = 120.00, steps = 120
13:28:23 [INFO] train episode 1353: reward = 113.00, steps = 113
13:28:30 [INFO] train episode 1354: reward = 132.00, steps = 132
13:28:40 [INFO] train episode 1355: reward = 171.00, steps = 171
13:28:46 [INFO] train episode 1356: reward = 112.00, steps = 112
13:28:54 [INFO] train episode 1357: reward = 162.00, steps = 162
13:29:03 [INFO] train episode 1358: reward = 162.00, steps = 162
13:29:10 [INFO] train episode 1359: reward = 131.00, steps = 131
13:29:19 [INFO] train episode 1360: reward = 153.00, steps = 153
13:29:24 [INFO] train episode 1361: reward = 105.00, steps = 105
13:29:32 [INFO] train episode 1362: reward = 145.00, steps = 145
13:29:42 [INFO] train episode 1363: reward = 173.00, steps = 173
13:29:50 [INFO] train episode 1364: reward = 147.00, steps = 147
13:29:59 [INFO] train episode 1365: reward = 179.00, steps = 179
13:30:10 [INFO] train episode 1366: reward = 200.00, steps = 200
13:30:19 [INFO] train episode 1367: reward = 168.00, steps = 168
13:30:27 [INFO] train episode 1368: reward = 144.00, steps = 144
13:30:33 [INFO] train episode 1369: reward = 110.00, steps = 110
13:30:39 [INFO] train episode 1370: reward = 102.00, steps = 102
13:30:46 [INFO] train episode 1371: reward = 136.00, steps = 136
13:30:57 [INFO] train episode 1372: reward = 200.00, steps = 200
13:31:06 [INFO] train episode 1373: reward = 165.00, steps = 165
13:31:16 [INFO] train episode 1374: reward = 187.00, steps = 187
13:31:23 [INFO] train episode 1375: reward = 126.00, steps = 126
13:31:30 [INFO] train episode 1376: reward = 124.00, steps = 124
13:31:36 [INFO] train episode 1377: reward = 104.00, steps = 104
13:31:44 [INFO] train episode 1378: reward = 163.00, steps = 163
13:31:51 [INFO] train episode 1379: reward = 126.00, steps = 126
13:31:58 [INFO] train episode 1380: reward = 120.00, steps = 120
13:32:04 [INFO] train episode 1381: reward = 106.00, steps = 106
13:32:10 [INFO] train episode 1382: reward = 115.00, steps = 115
13:32:19 [INFO] train episode 1383: reward = 166.00, steps = 166
13:32:26 [INFO] train episode 1384: reward = 123.00, steps = 123
13:32:37 [INFO] train episode 1385: reward = 200.00, steps = 200
13:32:45 [INFO] train episode 1386: reward = 154.00, steps = 154
13:32:53 [INFO] train episode 1387: reward = 132.00, steps = 132
13:33:03 [INFO] train episode 1388: reward = 200.00, steps = 200
13:33:13 [INFO] train episode 1389: reward = 169.00, steps = 169
13:33:23 [INFO] train episode 1390: reward = 200.00, steps = 200
13:33:30 [INFO] train episode 1391: reward = 114.00, steps = 114
13:33:37 [INFO] train episode 1392: reward = 132.00, steps = 132
13:33:45 [INFO] train episode 1393: reward = 148.00, steps = 148
13:33:52 [INFO] train episode 1394: reward = 132.00, steps = 132
13:34:03 [INFO] train episode 1395: reward = 189.00, steps = 189
13:34:13 [INFO] train episode 1396: reward = 152.00, steps = 152
13:34:24 [INFO] train episode 1397: reward = 158.00, steps = 158
13:34:29 [INFO] train episode 1398: reward = 90.00, steps = 90
13:34:42 [INFO] train episode 1399: reward = 200.00, steps = 200
13:34:52 [INFO] train episode 1400: reward = 149.00, steps = 149
13:35:02 [INFO] train episode 1401: reward = 143.00, steps = 143
13:35:12 [INFO] train episode 1402: reward = 163.00, steps = 163
13:35:19 [INFO] train episode 1403: reward = 109.00, steps = 109
13:35:29 [INFO] train episode 1404: reward = 149.00, steps = 149
13:35:37 [INFO] train episode 1405: reward = 134.00, steps = 134
13:35:43 [INFO] train episode 1406: reward = 98.00, steps = 98
13:35:52 [INFO] train episode 1407: reward = 144.00, steps = 144
13:36:01 [INFO] train episode 1408: reward = 146.00, steps = 146
13:36:11 [INFO] train episode 1409: reward = 170.00, steps = 170
13:36:21 [INFO] train episode 1410: reward = 157.00, steps = 157
13:36:31 [INFO] train episode 1411: reward = 146.00, steps = 146
13:36:44 [INFO] train episode 1412: reward = 189.00, steps = 189
13:36:52 [INFO] train episode 1413: reward = 112.00, steps = 112
13:37:01 [INFO] train episode 1414: reward = 162.00, steps = 162
13:37:09 [INFO] train episode 1415: reward = 133.00, steps = 133
13:37:17 [INFO] train episode 1416: reward = 153.00, steps = 153
13:37:24 [INFO] train episode 1417: reward = 122.00, steps = 122
13:37:31 [INFO] train episode 1418: reward = 119.00, steps = 119
13:37:41 [INFO] train episode 1419: reward = 175.00, steps = 175
13:37:50 [INFO] train episode 1420: reward = 181.00, steps = 181
13:37:58 [INFO] train episode 1421: reward = 152.00, steps = 152
13:38:04 [INFO] train episode 1422: reward = 134.00, steps = 134
13:38:12 [INFO] train episode 1423: reward = 158.00, steps = 158
13:38:20 [INFO] train episode 1424: reward = 149.00, steps = 149
13:38:26 [INFO] train episode 1425: reward = 132.00, steps = 132
13:38:34 [INFO] train episode 1426: reward = 156.00, steps = 156
13:38:44 [INFO] train episode 1427: reward = 173.00, steps = 173
13:38:51 [INFO] train episode 1428: reward = 122.00, steps = 122
13:39:04 [INFO] train episode 1429: reward = 200.00, steps = 200
13:39:14 [INFO] train episode 1430: reward = 135.00, steps = 135
13:39:24 [INFO] train episode 1431: reward = 115.00, steps = 115
13:39:33 [INFO] train episode 1432: reward = 119.00, steps = 119
13:39:47 [INFO] train episode 1433: reward = 196.00, steps = 196
13:39:56 [INFO] train episode 1434: reward = 157.00, steps = 157
13:40:08 [INFO] train episode 1435: reward = 200.00, steps = 200
13:40:16 [INFO] train episode 1436: reward = 152.00, steps = 152
13:40:27 [INFO] train episode 1437: reward = 188.00, steps = 188
13:40:37 [INFO] train episode 1438: reward = 167.00, steps = 167
13:40:46 [INFO] train episode 1439: reward = 128.00, steps = 128
13:40:54 [INFO] train episode 1440: reward = 156.00, steps = 156
13:41:01 [INFO] train episode 1441: reward = 125.00, steps = 125
13:41:08 [INFO] train episode 1442: reward = 128.00, steps = 128
13:41:18 [INFO] train episode 1443: reward = 178.00, steps = 178
13:41:28 [INFO] train episode 1444: reward = 181.00, steps = 181
13:41:35 [INFO] train episode 1445: reward = 148.00, steps = 148
13:41:41 [INFO] train episode 1446: reward = 104.00, steps = 104
13:41:50 [INFO] train episode 1447: reward = 184.00, steps = 184
13:41:55 [INFO] train episode 1448: reward = 117.00, steps = 117
13:42:02 [INFO] train episode 1449: reward = 136.00, steps = 136
13:42:09 [INFO] train episode 1450: reward = 140.00, steps = 140
13:42:17 [INFO] train episode 1451: reward = 150.00, steps = 150
13:42:24 [INFO] train episode 1452: reward = 134.00, steps = 134
13:42:31 [INFO] train episode 1453: reward = 155.00, steps = 155
13:42:40 [INFO] train episode 1454: reward = 170.00, steps = 170
13:42:48 [INFO] train episode 1455: reward = 168.00, steps = 168
13:42:57 [INFO] train episode 1456: reward = 188.00, steps = 188
13:43:02 [INFO] train episode 1457: reward = 105.00, steps = 105
13:43:11 [INFO] train episode 1458: reward = 169.00, steps = 169
13:43:17 [INFO] train episode 1459: reward = 137.00, steps = 137
13:43:24 [INFO] train episode 1460: reward = 132.00, steps = 132
13:43:33 [INFO] train episode 1461: reward = 185.00, steps = 185
13:43:41 [INFO] train episode 1462: reward = 160.00, steps = 160
13:43:46 [INFO] train episode 1463: reward = 121.00, steps = 121
13:43:56 [INFO] train episode 1464: reward = 200.00, steps = 200
13:44:01 [INFO] train episode 1465: reward = 98.00, steps = 98
13:44:09 [INFO] train episode 1466: reward = 159.00, steps = 159
13:44:15 [INFO] train episode 1467: reward = 126.00, steps = 126
13:44:21 [INFO] train episode 1468: reward = 128.00, steps = 128
13:44:31 [INFO] train episode 1469: reward = 200.00, steps = 200
13:44:38 [INFO] train episode 1470: reward = 140.00, steps = 140
13:44:48 [INFO] train episode 1471: reward = 200.00, steps = 200
13:44:54 [INFO] train episode 1472: reward = 124.00, steps = 124
13:45:02 [INFO] train episode 1473: reward = 164.00, steps = 164
13:45:08 [INFO] train episode 1474: reward = 121.00, steps = 121
13:45:14 [INFO] train episode 1475: reward = 115.00, steps = 115
13:45:21 [INFO] train episode 1476: reward = 152.00, steps = 152
13:45:27 [INFO] train episode 1477: reward = 112.00, steps = 112
13:45:35 [INFO] train episode 1478: reward = 173.00, steps = 173
13:45:43 [INFO] train episode 1479: reward = 146.00, steps = 146
13:45:50 [INFO] train episode 1480: reward = 141.00, steps = 141
13:45:56 [INFO] train episode 1481: reward = 122.00, steps = 122
13:46:05 [INFO] train episode 1482: reward = 200.00, steps = 200
13:46:12 [INFO] train episode 1483: reward = 128.00, steps = 128
13:46:22 [INFO] train episode 1484: reward = 198.00, steps = 198
13:46:31 [INFO] train episode 1485: reward = 200.00, steps = 200
13:46:39 [INFO] train episode 1486: reward = 165.00, steps = 165
13:46:46 [INFO] train episode 1487: reward = 138.00, steps = 138
13:46:56 [INFO] train episode 1488: reward = 200.00, steps = 200
13:47:06 [INFO] train episode 1489: reward = 195.00, steps = 195
13:47:15 [INFO] train episode 1490: reward = 193.00, steps = 193
13:47:24 [INFO] train episode 1491: reward = 177.00, steps = 177
13:47:34 [INFO] train episode 1492: reward = 187.00, steps = 187
13:47:42 [INFO] train episode 1493: reward = 161.00, steps = 161
13:47:48 [INFO] train episode 1494: reward = 111.00, steps = 111
13:47:56 [INFO] train episode 1495: reward = 148.00, steps = 148
13:48:06 [INFO] train episode 1496: reward = 200.00, steps = 200
13:48:14 [INFO] train episode 1497: reward = 157.00, steps = 157
13:48:22 [INFO] train episode 1498: reward = 152.00, steps = 152
13:48:30 [INFO] train episode 1499: reward = 140.00, steps = 140
13:48:37 [INFO] train episode 1500: reward = 136.00, steps = 136
13:48:45 [INFO] train episode 1501: reward = 149.00, steps = 149
13:48:55 [INFO] train episode 1502: reward = 200.00, steps = 200
13:49:06 [INFO] train episode 1503: reward = 200.00, steps = 200
13:49:15 [INFO] train episode 1504: reward = 174.00, steps = 174
13:49:23 [INFO] train episode 1505: reward = 161.00, steps = 161
13:49:30 [INFO] train episode 1506: reward = 137.00, steps = 137
13:49:38 [INFO] train episode 1507: reward = 139.00, steps = 139
13:49:44 [INFO] train episode 1508: reward = 122.00, steps = 122
13:49:55 [INFO] train episode 1509: reward = 200.00, steps = 200
13:50:05 [INFO] train episode 1510: reward = 200.00, steps = 200
13:50:12 [INFO] train episode 1511: reward = 127.00, steps = 127
13:50:18 [INFO] train episode 1512: reward = 125.00, steps = 125
13:50:26 [INFO] train episode 1513: reward = 152.00, steps = 152
13:50:33 [INFO] train episode 1514: reward = 118.00, steps = 118
13:50:43 [INFO] train episode 1515: reward = 197.00, steps = 197
13:50:52 [INFO] train episode 1516: reward = 169.00, steps = 169
13:51:01 [INFO] train episode 1517: reward = 183.00, steps = 183
13:51:11 [INFO] train episode 1518: reward = 189.00, steps = 189
13:51:17 [INFO] train episode 1519: reward = 110.00, steps = 110
13:51:25 [INFO] train episode 1520: reward = 152.00, steps = 152
13:51:34 [INFO] train episode 1521: reward = 176.00, steps = 176
13:51:43 [INFO] train episode 1522: reward = 162.00, steps = 162
13:51:50 [INFO] train episode 1523: reward = 143.00, steps = 143
13:51:56 [INFO] train episode 1524: reward = 113.00, steps = 113
13:52:05 [INFO] train episode 1525: reward = 179.00, steps = 179
13:52:14 [INFO] train episode 1526: reward = 163.00, steps = 163
13:52:23 [INFO] train episode 1527: reward = 170.00, steps = 170
13:52:34 [INFO] train episode 1528: reward = 200.00, steps = 200
13:52:43 [INFO] train episode 1529: reward = 173.00, steps = 173
13:52:51 [INFO] train episode 1530: reward = 160.00, steps = 160
13:52:58 [INFO] train episode 1531: reward = 140.00, steps = 140
13:53:05 [INFO] train episode 1532: reward = 126.00, steps = 126
13:53:12 [INFO] train episode 1533: reward = 144.00, steps = 144
13:53:19 [INFO] train episode 1534: reward = 132.00, steps = 132
13:53:29 [INFO] train episode 1535: reward = 178.00, steps = 178
13:53:39 [INFO] train episode 1536: reward = 189.00, steps = 189
13:53:49 [INFO] train episode 1537: reward = 200.00, steps = 200
13:53:59 [INFO] train episode 1538: reward = 200.00, steps = 200
13:54:06 [INFO] train episode 1539: reward = 123.00, steps = 123
13:54:12 [INFO] train episode 1540: reward = 113.00, steps = 113
13:54:19 [INFO] train episode 1541: reward = 141.00, steps = 141
13:54:27 [INFO] train episode 1542: reward = 146.00, steps = 146
13:54:38 [INFO] train episode 1543: reward = 200.00, steps = 200
13:54:45 [INFO] train episode 1544: reward = 137.00, steps = 137
13:54:54 [INFO] train episode 1545: reward = 186.00, steps = 186
13:55:03 [INFO] train episode 1546: reward = 159.00, steps = 159
13:55:13 [INFO] train episode 1547: reward = 199.00, steps = 199
13:55:21 [INFO] train episode 1548: reward = 144.00, steps = 144
13:55:28 [INFO] train episode 1549: reward = 141.00, steps = 141
13:55:39 [INFO] train episode 1550: reward = 200.00, steps = 200
13:55:49 [INFO] train episode 1551: reward = 200.00, steps = 200
13:56:00 [INFO] train episode 1552: reward = 200.00, steps = 200
13:56:10 [INFO] train episode 1553: reward = 200.00, steps = 200
13:56:19 [INFO] train episode 1554: reward = 174.00, steps = 174
13:56:26 [INFO] train episode 1555: reward = 129.00, steps = 129
13:56:33 [INFO] train episode 1556: reward = 140.00, steps = 140
13:56:44 [INFO] train episode 1557: reward = 200.00, steps = 200
13:56:50 [INFO] train episode 1558: reward = 122.00, steps = 122
13:57:01 [INFO] train episode 1559: reward = 200.00, steps = 200
13:57:11 [INFO] train episode 1560: reward = 200.00, steps = 200
13:57:18 [INFO] train episode 1561: reward = 123.00, steps = 123
13:57:27 [INFO] train episode 1562: reward = 178.00, steps = 178
13:57:33 [INFO] train episode 1563: reward = 106.00, steps = 106
13:57:40 [INFO] train episode 1564: reward = 141.00, steps = 141
13:57:50 [INFO] train episode 1565: reward = 200.00, steps = 200
13:57:58 [INFO] train episode 1566: reward = 145.00, steps = 145
13:58:05 [INFO] train episode 1567: reward = 139.00, steps = 139
13:58:13 [INFO] train episode 1568: reward = 148.00, steps = 148
13:58:22 [INFO] train episode 1569: reward = 169.00, steps = 169
13:58:29 [INFO] train episode 1570: reward = 140.00, steps = 140
13:58:39 [INFO] train episode 1571: reward = 176.00, steps = 176
13:58:46 [INFO] train episode 1572: reward = 139.00, steps = 139
13:58:53 [INFO] train episode 1573: reward = 129.00, steps = 129
13:59:01 [INFO] train episode 1574: reward = 146.00, steps = 146
13:59:08 [INFO] train episode 1575: reward = 132.00, steps = 132
13:59:14 [INFO] train episode 1576: reward = 121.00, steps = 121
13:59:21 [INFO] train episode 1577: reward = 136.00, steps = 136
13:59:32 [INFO] train episode 1578: reward = 200.00, steps = 200
13:59:37 [INFO] train episode 1579: reward = 107.00, steps = 107
13:59:44 [INFO] train episode 1580: reward = 132.00, steps = 132
13:59:54 [INFO] train episode 1581: reward = 181.00, steps = 181
13:59:59 [INFO] train episode 1582: reward = 109.00, steps = 109
14:00:09 [INFO] train episode 1583: reward = 184.00, steps = 184
14:00:19 [INFO] train episode 1584: reward = 193.00, steps = 193
14:00:26 [INFO] train episode 1585: reward = 132.00, steps = 132
14:00:34 [INFO] train episode 1586: reward = 157.00, steps = 157
14:00:43 [INFO] train episode 1587: reward = 158.00, steps = 158
14:00:53 [INFO] train episode 1588: reward = 184.00, steps = 184
14:01:03 [INFO] train episode 1589: reward = 200.00, steps = 200
14:01:10 [INFO] train episode 1590: reward = 129.00, steps = 129
14:01:17 [INFO] train episode 1591: reward = 144.00, steps = 144
14:01:25 [INFO] train episode 1592: reward = 148.00, steps = 148
14:01:35 [INFO] train episode 1593: reward = 188.00, steps = 188
14:01:42 [INFO] train episode 1594: reward = 123.00, steps = 123
14:01:50 [INFO] train episode 1595: reward = 160.00, steps = 160
14:01:56 [INFO] train episode 1596: reward = 111.00, steps = 111
14:02:04 [INFO] train episode 1597: reward = 145.00, steps = 145
14:02:14 [INFO] train episode 1598: reward = 195.00, steps = 195
14:02:24 [INFO] train episode 1599: reward = 200.00, steps = 200
14:02:33 [INFO] train episode 1600: reward = 164.00, steps = 164
14:02:39 [INFO] train episode 1601: reward = 122.00, steps = 122
14:02:47 [INFO] train episode 1602: reward = 151.00, steps = 151
14:02:58 [INFO] train episode 1603: reward = 200.00, steps = 200
14:03:04 [INFO] train episode 1604: reward = 118.00, steps = 118
14:03:12 [INFO] train episode 1605: reward = 141.00, steps = 141
14:03:20 [INFO] train episode 1606: reward = 162.00, steps = 162
14:03:28 [INFO] train episode 1607: reward = 145.00, steps = 145
14:03:37 [INFO] train episode 1608: reward = 178.00, steps = 178
14:03:45 [INFO] train episode 1609: reward = 145.00, steps = 145
14:03:55 [INFO] train episode 1610: reward = 200.00, steps = 200
14:04:02 [INFO] train episode 1611: reward = 138.00, steps = 138
14:04:10 [INFO] train episode 1612: reward = 144.00, steps = 144
14:04:20 [INFO] train episode 1613: reward = 200.00, steps = 200
14:04:28 [INFO] train episode 1614: reward = 138.00, steps = 138
14:04:38 [INFO] train episode 1615: reward = 187.00, steps = 187
14:04:44 [INFO] train episode 1616: reward = 115.00, steps = 115
14:04:50 [INFO] train episode 1617: reward = 119.00, steps = 119
14:04:58 [INFO] train episode 1618: reward = 150.00, steps = 150
14:05:09 [INFO] train episode 1619: reward = 200.00, steps = 200
14:05:19 [INFO] train episode 1620: reward = 200.00, steps = 200
14:05:30 [INFO] train episode 1621: reward = 200.00, steps = 200
14:05:38 [INFO] train episode 1622: reward = 155.00, steps = 155
14:05:45 [INFO] train episode 1623: reward = 142.00, steps = 142
14:05:53 [INFO] train episode 1624: reward = 140.00, steps = 140
14:05:59 [INFO] train episode 1625: reward = 132.00, steps = 132
14:06:06 [INFO] train episode 1626: reward = 120.00, steps = 120
14:06:13 [INFO] train episode 1627: reward = 132.00, steps = 132
14:06:23 [INFO] train episode 1628: reward = 200.00, steps = 200
14:06:29 [INFO] train episode 1629: reward = 113.00, steps = 113
14:06:40 [INFO] train episode 1630: reward = 195.00, steps = 195
14:06:45 [INFO] train episode 1631: reward = 106.00, steps = 106
14:06:51 [INFO] train episode 1632: reward = 105.00, steps = 105
14:07:01 [INFO] train episode 1633: reward = 189.00, steps = 189
14:07:09 [INFO] train episode 1634: reward = 166.00, steps = 166
14:07:19 [INFO] train episode 1635: reward = 179.00, steps = 179
14:07:27 [INFO] train episode 1636: reward = 154.00, steps = 154
14:07:35 [INFO] train episode 1637: reward = 151.00, steps = 151
14:07:45 [INFO] train episode 1638: reward = 200.00, steps = 200
14:07:56 [INFO] train episode 1639: reward = 200.00, steps = 200
14:08:02 [INFO] train episode 1640: reward = 118.00, steps = 118
14:08:13 [INFO] train episode 1641: reward = 200.00, steps = 200
14:08:23 [INFO] train episode 1642: reward = 191.00, steps = 191
14:08:30 [INFO] train episode 1643: reward = 138.00, steps = 138
14:08:37 [INFO] train episode 1644: reward = 133.00, steps = 133
14:08:44 [INFO] train episode 1645: reward = 127.00, steps = 127
14:08:53 [INFO] train episode 1646: reward = 178.00, steps = 178
14:08:58 [INFO] train episode 1647: reward = 97.00, steps = 97
14:09:08 [INFO] train episode 1648: reward = 186.00, steps = 186
14:09:16 [INFO] train episode 1649: reward = 155.00, steps = 155
14:09:22 [INFO] train episode 1650: reward = 117.00, steps = 117
14:09:32 [INFO] train episode 1651: reward = 179.00, steps = 179
14:09:40 [INFO] train episode 1652: reward = 154.00, steps = 154
14:09:49 [INFO] train episode 1653: reward = 140.00, steps = 140
14:09:58 [INFO] train episode 1654: reward = 175.00, steps = 175
14:10:09 [INFO] train episode 1655: reward = 199.00, steps = 199
14:10:18 [INFO] train episode 1656: reward = 186.00, steps = 186
14:10:29 [INFO] train episode 1657: reward = 200.00, steps = 200
14:10:36 [INFO] train episode 1658: reward = 136.00, steps = 136
14:10:44 [INFO] train episode 1659: reward = 144.00, steps = 144
14:10:52 [INFO] train episode 1660: reward = 157.00, steps = 157
14:10:59 [INFO] train episode 1661: reward = 136.00, steps = 136
14:11:07 [INFO] train episode 1662: reward = 132.00, steps = 132
14:11:14 [INFO] train episode 1663: reward = 138.00, steps = 138
14:11:23 [INFO] train episode 1664: reward = 165.00, steps = 165
14:11:33 [INFO] train episode 1665: reward = 195.00, steps = 195
14:11:43 [INFO] train episode 1666: reward = 200.00, steps = 200
14:11:50 [INFO] train episode 1667: reward = 132.00, steps = 132
14:12:01 [INFO] train episode 1668: reward = 200.00, steps = 200
14:12:11 [INFO] train episode 1669: reward = 187.00, steps = 187
14:12:18 [INFO] train episode 1670: reward = 139.00, steps = 139
14:12:25 [INFO] train episode 1671: reward = 140.00, steps = 140
14:12:33 [INFO] train episode 1672: reward = 156.00, steps = 156
14:12:41 [INFO] train episode 1673: reward = 134.00, steps = 134
14:12:51 [INFO] train episode 1674: reward = 200.00, steps = 200
14:12:57 [INFO] train episode 1675: reward = 118.00, steps = 118
14:13:04 [INFO] train episode 1676: reward = 120.00, steps = 120
14:13:12 [INFO] train episode 1677: reward = 168.00, steps = 168
14:13:22 [INFO] train episode 1678: reward = 163.00, steps = 163
14:13:30 [INFO] train episode 1679: reward = 137.00, steps = 137
14:13:38 [INFO] train episode 1680: reward = 158.00, steps = 158
14:13:47 [INFO] train episode 1681: reward = 161.00, steps = 161
14:13:55 [INFO] train episode 1682: reward = 134.00, steps = 134
14:14:06 [INFO] train episode 1683: reward = 200.00, steps = 200
14:14:14 [INFO] train episode 1684: reward = 153.00, steps = 153
14:14:23 [INFO] train episode 1685: reward = 166.00, steps = 166
14:14:36 [INFO] train episode 1686: reward = 200.00, steps = 200
14:14:46 [INFO] train episode 1687: reward = 171.00, steps = 171
14:14:52 [INFO] train episode 1688: reward = 110.00, steps = 110
14:14:59 [INFO] train episode 1689: reward = 126.00, steps = 126
14:15:07 [INFO] train episode 1690: reward = 148.00, steps = 148
14:15:17 [INFO] train episode 1691: reward = 170.00, steps = 170
14:15:24 [INFO] train episode 1692: reward = 121.00, steps = 121
14:15:36 [INFO] train episode 1693: reward = 200.00, steps = 200
14:15:48 [INFO] train episode 1694: reward = 178.00, steps = 178
14:15:54 [INFO] train episode 1695: reward = 116.00, steps = 116
14:16:01 [INFO] train episode 1696: reward = 126.00, steps = 126
14:16:08 [INFO] train episode 1697: reward = 122.00, steps = 122
14:16:15 [INFO] train episode 1698: reward = 124.00, steps = 124
14:16:26 [INFO] train episode 1699: reward = 200.00, steps = 200
14:16:37 [INFO] train episode 1700: reward = 193.00, steps = 193
14:16:50 [INFO] train episode 1701: reward = 200.00, steps = 200
14:17:02 [INFO] train episode 1702: reward = 189.00, steps = 189
14:17:12 [INFO] train episode 1703: reward = 200.00, steps = 200
14:17:22 [INFO] train episode 1704: reward = 177.00, steps = 177
14:17:32 [INFO] train episode 1705: reward = 200.00, steps = 200
14:17:41 [INFO] train episode 1706: reward = 161.00, steps = 161
14:17:47 [INFO] train episode 1707: reward = 128.00, steps = 128
14:17:55 [INFO] train episode 1708: reward = 133.00, steps = 133
14:18:05 [INFO] train episode 1709: reward = 200.00, steps = 200
14:18:13 [INFO] train episode 1710: reward = 150.00, steps = 150
14:18:21 [INFO] train episode 1711: reward = 142.00, steps = 142
14:18:27 [INFO] train episode 1712: reward = 125.00, steps = 125
14:18:34 [INFO] train episode 1713: reward = 131.00, steps = 131
14:18:43 [INFO] train episode 1714: reward = 160.00, steps = 160
14:18:49 [INFO] train episode 1715: reward = 127.00, steps = 127
14:18:55 [INFO] train episode 1716: reward = 113.00, steps = 113
14:19:03 [INFO] train episode 1717: reward = 137.00, steps = 137
14:19:10 [INFO] train episode 1718: reward = 136.00, steps = 136
14:19:20 [INFO] train episode 1719: reward = 200.00, steps = 200
14:19:27 [INFO] train episode 1720: reward = 128.00, steps = 128
14:19:35 [INFO] train episode 1721: reward = 140.00, steps = 140
14:19:42 [INFO] train episode 1722: reward = 135.00, steps = 135
14:19:50 [INFO] train episode 1723: reward = 150.00, steps = 150
14:20:00 [INFO] train episode 1724: reward = 200.00, steps = 200
14:20:11 [INFO] train episode 1725: reward = 200.00, steps = 200
14:20:21 [INFO] train episode 1726: reward = 200.00, steps = 200
14:20:32 [INFO] train episode 1727: reward = 200.00, steps = 200
14:20:39 [INFO] train episode 1728: reward = 127.00, steps = 127
14:20:47 [INFO] train episode 1729: reward = 147.00, steps = 147
14:20:54 [INFO] train episode 1730: reward = 134.00, steps = 134
14:21:03 [INFO] train episode 1731: reward = 178.00, steps = 178
14:21:10 [INFO] train episode 1732: reward = 133.00, steps = 133
14:21:18 [INFO] train episode 1733: reward = 136.00, steps = 136
14:21:27 [INFO] train episode 1734: reward = 173.00, steps = 173
14:21:35 [INFO] train episode 1735: reward = 153.00, steps = 153
14:21:42 [INFO] train episode 1736: reward = 134.00, steps = 134
14:21:52 [INFO] train episode 1737: reward = 193.00, steps = 193
14:21:59 [INFO] train episode 1738: reward = 136.00, steps = 136
14:22:08 [INFO] train episode 1739: reward = 164.00, steps = 164
14:22:14 [INFO] train episode 1740: reward = 104.00, steps = 104
14:22:21 [INFO] train episode 1741: reward = 129.00, steps = 129
14:22:28 [INFO] train episode 1742: reward = 139.00, steps = 139
14:22:37 [INFO] train episode 1743: reward = 183.00, steps = 183
14:22:44 [INFO] train episode 1744: reward = 124.00, steps = 124
14:22:52 [INFO] train episode 1745: reward = 144.00, steps = 144
14:22:58 [INFO] train episode 1746: reward = 111.00, steps = 111
14:23:04 [INFO] train episode 1747: reward = 123.00, steps = 123
14:23:15 [INFO] train episode 1748: reward = 195.00, steps = 195
14:23:24 [INFO] train episode 1749: reward = 167.00, steps = 167
14:23:30 [INFO] train episode 1750: reward = 113.00, steps = 113
14:23:39 [INFO] train episode 1751: reward = 155.00, steps = 155
14:23:45 [INFO] train episode 1752: reward = 110.00, steps = 110
14:23:55 [INFO] train episode 1753: reward = 189.00, steps = 189
14:24:04 [INFO] train episode 1754: reward = 156.00, steps = 156
14:24:12 [INFO] train episode 1755: reward = 135.00, steps = 135
14:24:22 [INFO] train episode 1756: reward = 188.00, steps = 188
14:24:31 [INFO] train episode 1757: reward = 162.00, steps = 162
14:24:39 [INFO] train episode 1758: reward = 157.00, steps = 157
14:24:45 [INFO] train episode 1759: reward = 112.00, steps = 112
14:24:53 [INFO] train episode 1760: reward = 158.00, steps = 158
14:25:00 [INFO] train episode 1761: reward = 131.00, steps = 131
14:25:07 [INFO] train episode 1762: reward = 136.00, steps = 136
14:25:17 [INFO] train episode 1763: reward = 182.00, steps = 182
14:25:25 [INFO] train episode 1764: reward = 158.00, steps = 158
14:25:36 [INFO] train episode 1765: reward = 200.00, steps = 200
14:25:43 [INFO] train episode 1766: reward = 134.00, steps = 134
14:25:54 [INFO] train episode 1767: reward = 200.00, steps = 200
14:26:00 [INFO] train episode 1768: reward = 120.00, steps = 120
14:26:08 [INFO] train episode 1769: reward = 155.00, steps = 155
14:26:19 [INFO] train episode 1770: reward = 200.00, steps = 200
14:26:26 [INFO] train episode 1771: reward = 144.00, steps = 144
14:26:34 [INFO] train episode 1772: reward = 127.00, steps = 127
14:26:48 [INFO] train episode 1773: reward = 200.00, steps = 200
14:27:00 [INFO] train episode 1774: reward = 200.00, steps = 200
14:27:11 [INFO] train episode 1775: reward = 200.00, steps = 200
14:27:18 [INFO] train episode 1776: reward = 116.00, steps = 116
14:27:29 [INFO] train episode 1777: reward = 200.00, steps = 200
14:27:40 [INFO] train episode 1778: reward = 187.00, steps = 187
14:27:47 [INFO] train episode 1779: reward = 118.00, steps = 118
14:27:59 [INFO] train episode 1780: reward = 200.00, steps = 200
14:28:07 [INFO] train episode 1781: reward = 130.00, steps = 130
14:28:17 [INFO] train episode 1782: reward = 161.00, steps = 161
14:28:26 [INFO] train episode 1783: reward = 156.00, steps = 156
14:28:38 [INFO] train episode 1784: reward = 200.00, steps = 200
14:28:47 [INFO] train episode 1785: reward = 134.00, steps = 134
14:28:58 [INFO] train episode 1786: reward = 200.00, steps = 200
14:29:09 [INFO] train episode 1787: reward = 167.00, steps = 167
14:29:22 [INFO] train episode 1788: reward = 200.00, steps = 200
14:29:30 [INFO] train episode 1789: reward = 149.00, steps = 149
14:29:39 [INFO] train episode 1790: reward = 141.00, steps = 141
14:29:50 [INFO] train episode 1791: reward = 164.00, steps = 164
14:30:03 [INFO] train episode 1792: reward = 200.00, steps = 200
14:30:11 [INFO] train episode 1793: reward = 140.00, steps = 140
14:30:23 [INFO] train episode 1794: reward = 200.00, steps = 200
14:30:30 [INFO] train episode 1795: reward = 125.00, steps = 125
14:30:40 [INFO] train episode 1796: reward = 194.00, steps = 194
14:30:51 [INFO] train episode 1797: reward = 200.00, steps = 200
14:30:59 [INFO] train episode 1798: reward = 149.00, steps = 149
14:31:06 [INFO] train episode 1799: reward = 134.00, steps = 134
14:31:13 [INFO] train episode 1800: reward = 131.00, steps = 131
14:31:24 [INFO] train episode 1801: reward = 198.00, steps = 198
14:31:34 [INFO] train episode 1802: reward = 148.00, steps = 148
14:31:45 [INFO] train episode 1803: reward = 199.00, steps = 199
14:31:54 [INFO] train episode 1804: reward = 142.00, steps = 142
14:32:03 [INFO] train episode 1805: reward = 170.00, steps = 170
14:32:10 [INFO] train episode 1806: reward = 127.00, steps = 127
14:32:23 [INFO] train episode 1807: reward = 200.00, steps = 200
14:32:33 [INFO] train episode 1808: reward = 160.00, steps = 160
14:32:43 [INFO] train episode 1809: reward = 170.00, steps = 170
14:32:54 [INFO] train episode 1810: reward = 200.00, steps = 200
14:33:06 [INFO] train episode 1811: reward = 200.00, steps = 200
14:33:17 [INFO] train episode 1812: reward = 182.00, steps = 182
14:33:27 [INFO] train episode 1813: reward = 176.00, steps = 176
14:33:37 [INFO] train episode 1814: reward = 179.00, steps = 179
14:33:45 [INFO] train episode 1815: reward = 144.00, steps = 144
14:33:57 [INFO] train episode 1816: reward = 200.00, steps = 200
14:34:05 [INFO] train episode 1817: reward = 142.00, steps = 142
14:34:14 [INFO] train episode 1818: reward = 161.00, steps = 161
14:34:24 [INFO] train episode 1819: reward = 176.00, steps = 176
14:34:35 [INFO] train episode 1820: reward = 192.00, steps = 192
14:34:46 [INFO] train episode 1821: reward = 193.00, steps = 193
14:34:55 [INFO] train episode 1822: reward = 159.00, steps = 159
14:35:07 [INFO] train episode 1823: reward = 200.00, steps = 200
14:35:16 [INFO] train episode 1824: reward = 158.00, steps = 158
14:35:23 [INFO] train episode 1825: reward = 128.00, steps = 128
14:35:31 [INFO] train episode 1826: reward = 135.00, steps = 135
14:35:38 [INFO] train episode 1827: reward = 141.00, steps = 141
14:35:50 [INFO] train episode 1828: reward = 200.00, steps = 200
14:36:01 [INFO] train episode 1829: reward = 200.00, steps = 200
14:36:12 [INFO] train episode 1830: reward = 193.00, steps = 193
14:36:24 [INFO] train episode 1831: reward = 200.00, steps = 200
14:36:32 [INFO] train episode 1832: reward = 150.00, steps = 150
14:36:40 [INFO] train episode 1833: reward = 148.00, steps = 148
14:36:49 [INFO] train episode 1834: reward = 156.00, steps = 156
14:36:57 [INFO] train episode 1835: reward = 137.00, steps = 137
14:37:08 [INFO] train episode 1836: reward = 186.00, steps = 186
14:37:17 [INFO] train episode 1837: reward = 165.00, steps = 165
14:37:25 [INFO] train episode 1838: reward = 141.00, steps = 141
14:37:32 [INFO] train episode 1839: reward = 126.00, steps = 126
14:37:41 [INFO] train episode 1840: reward = 165.00, steps = 165
14:37:52 [INFO] train episode 1841: reward = 200.00, steps = 200
14:38:04 [INFO] train episode 1842: reward = 200.00, steps = 200
14:38:13 [INFO] train episode 1843: reward = 180.00, steps = 180
14:38:25 [INFO] train episode 1844: reward = 198.00, steps = 198
14:38:36 [INFO] train episode 1845: reward = 188.00, steps = 188
14:38:43 [INFO] train episode 1846: reward = 125.00, steps = 125
14:38:50 [INFO] train episode 1847: reward = 119.00, steps = 119
14:38:58 [INFO] train episode 1848: reward = 151.00, steps = 151
14:39:06 [INFO] train episode 1849: reward = 131.00, steps = 131
14:39:15 [INFO] train episode 1850: reward = 174.00, steps = 174
14:39:25 [INFO] train episode 1851: reward = 165.00, steps = 165
14:39:33 [INFO] train episode 1852: reward = 158.00, steps = 158
14:39:45 [INFO] train episode 1853: reward = 200.00, steps = 200
14:39:53 [INFO] train episode 1854: reward = 151.00, steps = 151
14:40:01 [INFO] train episode 1855: reward = 144.00, steps = 144
14:40:12 [INFO] train episode 1856: reward = 200.00, steps = 200
14:40:21 [INFO] train episode 1857: reward = 161.00, steps = 161
14:40:29 [INFO] train episode 1858: reward = 141.00, steps = 141
14:40:40 [INFO] train episode 1859: reward = 200.00, steps = 200
14:40:49 [INFO] train episode 1860: reward = 150.00, steps = 150
14:40:59 [INFO] train episode 1861: reward = 169.00, steps = 169
14:41:08 [INFO] train episode 1862: reward = 144.00, steps = 144
14:41:19 [INFO] train episode 1863: reward = 200.00, steps = 200
14:41:28 [INFO] train episode 1864: reward = 168.00, steps = 168
14:41:39 [INFO] train episode 1865: reward = 200.00, steps = 200
14:41:49 [INFO] train episode 1866: reward = 200.00, steps = 200
14:41:57 [INFO] train episode 1867: reward = 145.00, steps = 145
14:42:07 [INFO] train episode 1868: reward = 191.00, steps = 191
14:42:17 [INFO] train episode 1869: reward = 177.00, steps = 177
14:42:25 [INFO] train episode 1870: reward = 149.00, steps = 149
14:42:36 [INFO] train episode 1871: reward = 200.00, steps = 200
14:42:45 [INFO] train episode 1872: reward = 164.00, steps = 164
14:42:55 [INFO] train episode 1873: reward = 160.00, steps = 160
14:43:04 [INFO] train episode 1874: reward = 165.00, steps = 165
14:43:15 [INFO] train episode 1875: reward = 200.00, steps = 200
14:43:26 [INFO] train episode 1876: reward = 200.00, steps = 200
14:43:38 [INFO] train episode 1877: reward = 200.00, steps = 200
14:43:49 [INFO] train episode 1878: reward = 196.00, steps = 196
14:44:01 [INFO] train episode 1879: reward = 195.00, steps = 195
14:44:13 [INFO] train episode 1880: reward = 200.00, steps = 200
14:44:23 [INFO] train episode 1881: reward = 177.00, steps = 177
14:44:31 [INFO] train episode 1882: reward = 144.00, steps = 144
14:44:39 [INFO] train episode 1883: reward = 146.00, steps = 146
14:44:50 [INFO] train episode 1884: reward = 179.00, steps = 179
14:45:02 [INFO] train episode 1885: reward = 176.00, steps = 176
14:45:13 [INFO] train episode 1886: reward = 198.00, steps = 198
14:45:24 [INFO] train episode 1887: reward = 200.00, steps = 200
14:45:34 [INFO] train episode 1888: reward = 171.00, steps = 171
14:45:43 [INFO] train episode 1889: reward = 164.00, steps = 164
14:45:54 [INFO] train episode 1890: reward = 179.00, steps = 179
14:46:03 [INFO] train episode 1891: reward = 156.00, steps = 156
14:46:12 [INFO] train episode 1892: reward = 150.00, steps = 150
14:46:24 [INFO] train episode 1893: reward = 197.00, steps = 197
14:46:31 [INFO] train episode 1894: reward = 139.00, steps = 139
14:46:43 [INFO] train episode 1895: reward = 200.00, steps = 200
14:46:54 [INFO] train episode 1896: reward = 196.00, steps = 196
14:47:04 [INFO] train episode 1897: reward = 200.00, steps = 200
14:47:15 [INFO] train episode 1898: reward = 200.00, steps = 200
14:47:22 [INFO] train episode 1899: reward = 126.00, steps = 126
14:47:29 [INFO] train episode 1900: reward = 134.00, steps = 134
14:47:40 [INFO] train episode 1901: reward = 192.00, steps = 192
14:47:50 [INFO] train episode 1902: reward = 168.00, steps = 168
14:47:59 [INFO] train episode 1903: reward = 160.00, steps = 160
14:48:10 [INFO] train episode 1904: reward = 193.00, steps = 193
14:48:18 [INFO] train episode 1905: reward = 160.00, steps = 160
14:48:28 [INFO] train episode 1906: reward = 186.00, steps = 186
14:48:35 [INFO] train episode 1907: reward = 126.00, steps = 126
14:48:44 [INFO] train episode 1908: reward = 168.00, steps = 168
14:48:56 [INFO] train episode 1909: reward = 200.00, steps = 200
14:49:07 [INFO] train episode 1910: reward = 177.00, steps = 177
14:49:16 [INFO] train episode 1911: reward = 160.00, steps = 160
14:49:27 [INFO] train episode 1912: reward = 200.00, steps = 200
14:49:34 [INFO] train episode 1913: reward = 129.00, steps = 129
14:49:43 [INFO] train episode 1914: reward = 161.00, steps = 161
14:49:54 [INFO] train episode 1915: reward = 200.00, steps = 200
14:50:03 [INFO] train episode 1916: reward = 144.00, steps = 144
14:50:13 [INFO] train episode 1917: reward = 171.00, steps = 171
14:50:24 [INFO] train episode 1918: reward = 169.00, steps = 169
14:50:33 [INFO] train episode 1919: reward = 154.00, steps = 154
14:50:42 [INFO] train episode 1920: reward = 155.00, steps = 155
14:50:54 [INFO] train episode 1921: reward = 200.00, steps = 200
14:51:03 [INFO] train episode 1922: reward = 150.00, steps = 150
14:51:14 [INFO] train episode 1923: reward = 189.00, steps = 189
14:51:28 [INFO] train episode 1924: reward = 200.00, steps = 200
14:51:40 [INFO] train episode 1925: reward = 179.00, steps = 179
14:51:52 [INFO] train episode 1926: reward = 159.00, steps = 159
14:52:05 [INFO] train episode 1927: reward = 170.00, steps = 170
14:52:20 [INFO] train episode 1928: reward = 200.00, steps = 200
14:52:34 [INFO] train episode 1929: reward = 185.00, steps = 185
14:52:46 [INFO] train episode 1930: reward = 164.00, steps = 164
14:52:57 [INFO] train episode 1931: reward = 200.00, steps = 200
14:53:08 [INFO] train episode 1932: reward = 186.00, steps = 186
14:53:17 [INFO] train episode 1933: reward = 166.00, steps = 166
14:53:28 [INFO] train episode 1934: reward = 192.00, steps = 192
14:53:39 [INFO] train episode 1935: reward = 200.00, steps = 200
14:53:47 [INFO] train episode 1936: reward = 142.00, steps = 142
14:53:57 [INFO] train episode 1937: reward = 177.00, steps = 177
14:54:08 [INFO] train episode 1938: reward = 191.00, steps = 191
14:54:19 [INFO] train episode 1939: reward = 200.00, steps = 200
14:54:30 [INFO] train episode 1940: reward = 200.00, steps = 200
14:54:41 [INFO] train episode 1941: reward = 200.00, steps = 200
14:54:50 [INFO] train episode 1942: reward = 174.00, steps = 174
14:55:00 [INFO] train episode 1943: reward = 175.00, steps = 175
14:55:11 [INFO] train episode 1944: reward = 200.00, steps = 200
14:55:19 [INFO] train episode 1945: reward = 160.00, steps = 160
14:55:29 [INFO] train episode 1946: reward = 167.00, steps = 167
14:55:36 [INFO] train episode 1947: reward = 134.00, steps = 134
14:55:46 [INFO] train episode 1948: reward = 174.00, steps = 174
14:55:56 [INFO] train episode 1949: reward = 200.00, steps = 200
14:56:07 [INFO] train episode 1950: reward = 200.00, steps = 200
14:56:16 [INFO] train episode 1951: reward = 159.00, steps = 159
14:56:24 [INFO] train episode 1952: reward = 153.00, steps = 153
14:56:33 [INFO] train episode 1953: reward = 178.00, steps = 178
14:56:43 [INFO] train episode 1954: reward = 162.00, steps = 162
14:56:53 [INFO] train episode 1955: reward = 171.00, steps = 171
14:57:04 [INFO] train episode 1956: reward = 200.00, steps = 200
14:57:16 [INFO] train episode 1957: reward = 200.00, steps = 200
14:57:25 [INFO] train episode 1958: reward = 160.00, steps = 160
14:57:36 [INFO] train episode 1959: reward = 200.00, steps = 200
14:57:44 [INFO] train episode 1960: reward = 145.00, steps = 145
14:57:53 [INFO] train episode 1961: reward = 162.00, steps = 162
14:58:03 [INFO] train episode 1962: reward = 200.00, steps = 200
14:58:12 [INFO] train episode 1963: reward = 155.00, steps = 155
14:58:20 [INFO] train episode 1964: reward = 159.00, steps = 159
14:58:30 [INFO] train episode 1965: reward = 174.00, steps = 174
14:58:42 [INFO] train episode 1966: reward = 200.00, steps = 200
14:58:53 [INFO] train episode 1967: reward = 199.00, steps = 199
14:59:03 [INFO] train episode 1968: reward = 200.00, steps = 200
14:59:14 [INFO] train episode 1969: reward = 200.00, steps = 200
14:59:22 [INFO] train episode 1970: reward = 143.00, steps = 143
14:59:32 [INFO] train episode 1971: reward = 200.00, steps = 200
14:59:43 [INFO] train episode 1972: reward = 200.00, steps = 200
14:59:54 [INFO] train episode 1973: reward = 200.00, steps = 200
15:00:05 [INFO] train episode 1974: reward = 200.00, steps = 200
15:00:16 [INFO] train episode 1975: reward = 200.00, steps = 200
15:00:26 [INFO] train episode 1976: reward = 156.00, steps = 156
15:00:39 [INFO] train episode 1977: reward = 200.00, steps = 200
15:00:47 [INFO] train episode 1978: reward = 141.00, steps = 141
15:00:57 [INFO] train episode 1979: reward = 192.00, steps = 192
15:01:09 [INFO] train episode 1980: reward = 199.00, steps = 199
15:01:17 [INFO] train episode 1981: reward = 152.00, steps = 152
15:01:26 [INFO] train episode 1982: reward = 166.00, steps = 166
15:01:37 [INFO] train episode 1983: reward = 200.00, steps = 200
15:01:48 [INFO] train episode 1984: reward = 200.00, steps = 200
15:01:58 [INFO] train episode 1985: reward = 200.00, steps = 200
15:02:09 [INFO] train episode 1986: reward = 198.00, steps = 198
15:02:21 [INFO] train episode 1987: reward = 200.00, steps = 200
15:02:31 [INFO] train episode 1988: reward = 169.00, steps = 169
15:02:45 [INFO] train episode 1989: reward = 200.00, steps = 200
15:02:59 [INFO] train episode 1990: reward = 200.00, steps = 200
15:03:11 [INFO] train episode 1991: reward = 200.00, steps = 200
15:03:22 [INFO] train episode 1992: reward = 194.00, steps = 194
15:03:22 [INFO] ==== test ====
15:03:34 [INFO] test episode 0: reward = 200.00, steps = 200
15:03:45 [INFO] test episode 1: reward = 200.00, steps = 200
15:03:58 [INFO] test episode 2: reward = 196.00, steps = 196
15:04:08 [INFO] test episode 3: reward = 155.00, steps = 155
15:04:21 [INFO] test episode 4: reward = 172.00, steps = 172
15:04:34 [INFO] test episode 5: reward = 189.00, steps = 189
15:04:46 [INFO] test episode 6: reward = 200.00, steps = 200
15:04:58 [INFO] test episode 7: reward = 200.00, steps = 200
15:05:10 [INFO] test episode 8: reward = 200.00, steps = 200
15:05:22 [INFO] test episode 9: reward = 200.00, steps = 200
15:05:33 [INFO] test episode 10: reward = 200.00, steps = 200
15:05:45 [INFO] test episode 11: reward = 200.00, steps = 200
15:05:56 [INFO] test episode 12: reward = 200.00, steps = 200
15:06:07 [INFO] test episode 13: reward = 200.00, steps = 200
15:06:18 [INFO] test episode 14: reward = 186.00, steps = 186
15:06:29 [INFO] test episode 15: reward = 200.00, steps = 200
15:06:40 [INFO] test episode 16: reward = 200.00, steps = 200
15:06:56 [INFO] test episode 17: reward = 200.00, steps = 200
15:07:09 [INFO] test episode 18: reward = 186.00, steps = 186
15:07:22 [INFO] test episode 19: reward = 174.00, steps = 174
15:07:34 [INFO] test episode 20: reward = 175.00, steps = 175
15:07:48 [INFO] test episode 21: reward = 195.00, steps = 195
15:08:01 [INFO] test episode 22: reward = 200.00, steps = 200
15:08:12 [INFO] test episode 23: reward = 200.00, steps = 200
15:08:24 [INFO] test episode 24: reward = 176.00, steps = 176
15:08:34 [INFO] test episode 25: reward = 200.00, steps = 200
15:08:46 [INFO] test episode 26: reward = 200.00, steps = 200
15:08:57 [INFO] test episode 27: reward = 200.00, steps = 200
15:09:08 [INFO] test episode 28: reward = 200.00, steps = 200
15:09:19 [INFO] test episode 29: reward = 187.00, steps = 187
15:09:31 [INFO] test episode 30: reward = 198.00, steps = 198
15:09:41 [INFO] test episode 31: reward = 200.00, steps = 200
15:09:51 [INFO] test episode 32: reward = 200.00, steps = 200
15:10:02 [INFO] test episode 33: reward = 200.00, steps = 200
15:10:12 [INFO] test episode 34: reward = 196.00, steps = 196
15:10:21 [INFO] test episode 35: reward = 176.00, steps = 176
15:10:32 [INFO] test episode 36: reward = 200.00, steps = 200
15:10:42 [INFO] test episode 37: reward = 200.00, steps = 200
15:10:53 [INFO] test episode 38: reward = 200.00, steps = 200
15:11:03 [INFO] test episode 39: reward = 172.00, steps = 172
15:11:13 [INFO] test episode 40: reward = 200.00, steps = 200
15:11:24 [INFO] test episode 41: reward = 200.00, steps = 200
15:11:34 [INFO] test episode 42: reward = 200.00, steps = 200
15:11:45 [INFO] test episode 43: reward = 200.00, steps = 200
15:11:53 [INFO] test episode 44: reward = 172.00, steps = 172
15:12:04 [INFO] test episode 45: reward = 200.00, steps = 200
15:12:15 [INFO] test episode 46: reward = 200.00, steps = 200
15:12:25 [INFO] test episode 47: reward = 200.00, steps = 200
15:12:34 [INFO] test episode 48: reward = 177.00, steps = 177
15:12:46 [INFO] test episode 49: reward = 200.00, steps = 200
15:12:56 [INFO] test episode 50: reward = 200.00, steps = 200
15:13:07 [INFO] test episode 51: reward = 200.00, steps = 200
15:13:17 [INFO] test episode 52: reward = 200.00, steps = 200
15:13:28 [INFO] test episode 53: reward = 200.00, steps = 200
15:13:38 [INFO] test episode 54: reward = 200.00, steps = 200
15:13:48 [INFO] test episode 55: reward = 200.00, steps = 200
15:13:59 [INFO] test episode 56: reward = 200.00, steps = 200
15:14:09 [INFO] test episode 57: reward = 200.00, steps = 200
15:14:20 [INFO] test episode 58: reward = 200.00, steps = 200
15:14:30 [INFO] test episode 59: reward = 200.00, steps = 200
15:14:41 [INFO] test episode 60: reward = 200.00, steps = 200
15:14:51 [INFO] test episode 61: reward = 200.00, steps = 200
15:15:02 [INFO] test episode 62: reward = 200.00, steps = 200
15:15:12 [INFO] test episode 63: reward = 200.00, steps = 200
15:15:23 [INFO] test episode 64: reward = 200.00, steps = 200
15:15:33 [INFO] test episode 65: reward = 200.00, steps = 200
15:15:44 [INFO] test episode 66: reward = 200.00, steps = 200
15:15:54 [INFO] test episode 67: reward = 200.00, steps = 200
15:16:04 [INFO] test episode 68: reward = 200.00, steps = 200
15:16:15 [INFO] test episode 69: reward = 200.00, steps = 200
15:16:24 [INFO] test episode 70: reward = 181.00, steps = 181
15:16:35 [INFO] test episode 71: reward = 200.00, steps = 200
15:16:45 [INFO] test episode 72: reward = 200.00, steps = 200
15:16:56 [INFO] test episode 73: reward = 200.00, steps = 200
15:17:06 [INFO] test episode 74: reward = 200.00, steps = 200
15:17:15 [INFO] test episode 75: reward = 173.00, steps = 173
15:17:26 [INFO] test episode 76: reward = 200.00, steps = 200
15:17:36 [INFO] test episode 77: reward = 200.00, steps = 200
15:17:46 [INFO] test episode 78: reward = 200.00, steps = 200
15:17:57 [INFO] test episode 79: reward = 200.00, steps = 200
15:18:07 [INFO] test episode 80: reward = 200.00, steps = 200
15:18:17 [INFO] test episode 81: reward = 186.00, steps = 186
15:18:27 [INFO] test episode 82: reward = 200.00, steps = 200
15:18:37 [INFO] test episode 83: reward = 187.00, steps = 187
15:18:48 [INFO] test episode 84: reward = 200.00, steps = 200
15:18:58 [INFO] test episode 85: reward = 200.00, steps = 200
15:19:09 [INFO] test episode 86: reward = 200.00, steps = 200
15:19:19 [INFO] test episode 87: reward = 200.00, steps = 200
15:19:30 [INFO] test episode 88: reward = 200.00, steps = 200
15:19:40 [INFO] test episode 89: reward = 200.00, steps = 200
15:19:51 [INFO] test episode 90: reward = 200.00, steps = 200
15:20:01 [INFO] test episode 91: reward = 200.00, steps = 200
15:20:11 [INFO] test episode 92: reward = 197.00, steps = 197
15:20:22 [INFO] test episode 93: reward = 200.00, steps = 200
15:20:32 [INFO] test episode 94: reward = 200.00, steps = 200
15:20:43 [INFO] test episode 95: reward = 200.00, steps = 200
15:20:53 [INFO] test episode 96: reward = 200.00, steps = 200
15:21:03 [INFO] test episode 97: reward = 189.00, steps = 189
15:21:14 [INFO] test episode 98: reward = 200.00, steps = 200
15:21:24 [INFO] test episode 99: reward = 200.00, steps = 200
15:21:24 [INFO] average episode reward = 195.95 ± 8.97

env.close()