Use PPO to Play Acrobot-v1

PyTorch version

%matplotlib inline

import sys
import logging
import itertools

import numpy as np
np.random.seed(0)
import pandas as pd
import scipy.signal as signal
import gym
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim
import torch.distributions as distributions
torch.manual_seed(0)

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')

env = gym.make('Acrobot-v1')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

22:56:35 [INFO] env: <AcrobotEnv<Acrobot-v1>>
22:56:35 [INFO] action_space: Discrete(3)
22:56:35 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)
22:56:35 [INFO] reward_range: (-inf, inf)
22:56:35 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}
22:56:35 [INFO] _max_episode_steps: 500
22:56:35 [INFO] _elapsed_steps: None
22:56:35 [INFO] id: Acrobot-v1
22:56:35 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv
22:56:35 [INFO] reward_threshold: -100.0
22:56:35 [INFO] nondeterministic: False
22:56:35 [INFO] max_episode_steps: 500
22:56:35 [INFO] _kwargs: {}
22:56:35 [INFO] _env_name: Acrobot

class PPOReplayer:
    def __init__(self):
        self.fields = ['state', 'action', 'prob', 'advantage', 'return']
        self.memory = pd.DataFrame(columns=self.fields)

    def store(self, df):
        if self.memory.empty:
            self.memory = df[self.fields]
        else:
            self.memory = pd.concat([self.memory, df[self.fields]], ignore_index=True)

    def sample(self, size):
        indices = np.random.choice(self.memory.shape[0], size=size)
        return (np.stack(self.memory.loc[indices, field]) for field in
                self.fields)

class PPOAgent:
    def __init__(self, env):
        self.gamma = 0.99

        self.replayer = PPOReplayer()

        self.actor_net = self.build_net(
                input_size=env.observation_space.shape[0],
                hidden_sizes=[100,],
                output_size=env.action_space.n, output_activator=nn.Softmax(1))
        self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 0.001)
        self.critic_net = self.build_net(
                input_size=env.observation_space.shape[0],
                hidden_sizes=[100,])
        self.critic_optimizer = optim.Adam(self.critic_net.parameters(), 0.002)
        self.critic_loss = nn.MSELoss()

    def build_net(self, input_size, hidden_sizes, output_size=1,
            output_activator=None):
        layers = []
        for input_size, output_size in zip(
                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):
            layers.append(nn.Linear(input_size, output_size))
            layers.append(nn.ReLU())
        layers = layers[:-1]
        if output_activator:
            layers.append(output_activator)
        net = nn.Sequential(*layers)
        return net

    def reset(self, mode=None):
        self.mode = mode
        if self.mode == 'train':
            self.trajectory = []

    def step(self, observation, reward, terminated):
        state_tensor = torch.as_tensor(observation, dtype=torch.float).unsqueeze(0)
        prob_tensor = self.actor_net(state_tensor)
        action_tensor = distributions.Categorical(prob_tensor).sample()
        action = action_tensor.numpy()[0]
        if self.mode == 'train':
            self.trajectory += [observation, reward, terminated, action]
        return action

    def close(self):
        if self.mode == 'train':
            self.save_trajectory_to_replayer()
            if len(self.replayer.memory) >= 1000:
                for batch in range(5):  # learn multiple times
                    self.learn()
                self.replayer = PPOReplayer()
                        # reset replayer after the agent changes itself

    def save_trajectory_to_replayer(self):
        df = pd.DataFrame(
                np.array(self.trajectory, dtype=object).reshape(-1, 4),
                columns=['state', 'reward', 'terminated', 'action'])
        state_tensor = torch.as_tensor(np.stack(df['state']), dtype=torch.float)
        action_tensor = torch.as_tensor(df['action'], dtype=torch.long)
        v_tensor = self.critic_net(state_tensor)
        df['v'] = v_tensor.detach().numpy()
        prob_tensor = self.actor_net(state_tensor)
        pi_tensor = prob_tensor.gather(-1, action_tensor.unsqueeze(1)).squeeze(1)
        df['prob'] = pi_tensor.detach().numpy()
        df['next_v'] = df['v'].shift(-1).fillna(0.)
        df['u'] = df['reward'] + self.gamma * df['next_v']
        df['delta'] = df['u'] - df['v']
        df['advantage'] = signal.lfilter([1.,], [1., -self.gamma],
                df['delta'][::-1])[::-1]
        df['return'] = signal.lfilter([1.,], [1., -self.gamma],
                df['reward'][::-1])[::-1]
        self.replayer.store(df)

    def learn(self):
        states, actions, old_pis, advantages, returns = \
                self.replayer.sample(size=64)
        state_tensor = torch.as_tensor(states, dtype=torch.float)
        action_tensor = torch.as_tensor(actions, dtype=torch.long)
        old_pi_tensor = torch.as_tensor(old_pis, dtype=torch.float)
        advantage_tensor = torch.as_tensor(advantages, dtype=torch.float)
        return_tensor = torch.as_tensor(returns, dtype=torch.float).unsqueeze(1)

        # update actor
        all_pi_tensor = self.actor_net(state_tensor)
        pi_tensor = all_pi_tensor.gather(1, action_tensor.unsqueeze(1)).squeeze(1)
        surrogate_advantage_tensor = (pi_tensor / old_pi_tensor) * \
                advantage_tensor
        clip_times_advantage_tensor = 0.1 * surrogate_advantage_tensor
        max_surrogate_advantage_tensor = advantage_tensor + \
                torch.where(advantage_tensor > 0.,
                clip_times_advantage_tensor, -clip_times_advantage_tensor)
        clipped_surrogate_advantage_tensor = torch.min(
                surrogate_advantage_tensor, max_surrogate_advantage_tensor)
        actor_loss_tensor = -clipped_surrogate_advantage_tensor.mean()
        self.actor_optimizer.zero_grad()
        actor_loss_tensor.backward()
        self.actor_optimizer.step()

        # update critic
        pred_tensor = self.critic_net(state_tensor)
        critic_loss_tensor = self.critic_loss(pred_tensor, return_tensor)
        self.critic_optimizer.zero_grad()
        critic_loss_tensor.backward()
        self.critic_optimizer.step()


agent = PPOAgent(env)

def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== train ====')
episode_rewards = []
for episode in itertools.count():
    episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
            mode='train')
    episode_rewards.append(episode_reward)
    logging.info('train episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > -120:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

22:56:35 [INFO] ==== train ====
22:56:36 [INFO] NumExpr defaulting to 8 threads.
22:56:36 [INFO] train episode 0: reward = -500.00, steps = 500
22:56:37 [INFO] train episode 1: reward = -500.00, steps = 500
22:56:38 [INFO] train episode 2: reward = -500.00, steps = 500
22:56:40 [INFO] train episode 3: reward = -500.00, steps = 500
22:56:41 [INFO] train episode 4: reward = -500.00, steps = 500
22:56:42 [INFO] train episode 5: reward = -500.00, steps = 500
22:56:43 [INFO] train episode 6: reward = -500.00, steps = 500
22:56:45 [INFO] train episode 7: reward = -500.00, steps = 500
22:56:46 [INFO] train episode 8: reward = -500.00, steps = 500
22:56:47 [INFO] train episode 9: reward = -500.00, steps = 500
22:56:48 [INFO] train episode 10: reward = -500.00, steps = 500
22:56:50 [INFO] train episode 11: reward = -500.00, steps = 500
22:56:51 [INFO] train episode 12: reward = -500.00, steps = 500
22:56:52 [INFO] train episode 13: reward = -500.00, steps = 500
22:56:53 [INFO] train episode 14: reward = -500.00, steps = 500
22:56:55 [INFO] train episode 15: reward = -500.00, steps = 500
22:56:56 [INFO] train episode 16: reward = -500.00, steps = 500
22:56:57 [INFO] train episode 17: reward = -500.00, steps = 500
22:56:58 [INFO] train episode 18: reward = -500.00, steps = 500
22:57:00 [INFO] train episode 19: reward = -500.00, steps = 500
22:57:01 [INFO] train episode 20: reward = -500.00, steps = 500
22:57:02 [INFO] train episode 21: reward = -500.00, steps = 500
22:57:03 [INFO] train episode 22: reward = -500.00, steps = 500
22:57:05 [INFO] train episode 23: reward = -500.00, steps = 500
22:57:06 [INFO] train episode 24: reward = -500.00, steps = 500
22:57:07 [INFO] train episode 25: reward = -500.00, steps = 500
22:57:08 [INFO] train episode 26: reward = -395.00, steps = 396
22:57:09 [INFO] train episode 27: reward = -500.00, steps = 500
22:57:10 [INFO] train episode 28: reward = -500.00, steps = 500
22:57:11 [INFO] train episode 29: reward = -500.00, steps = 500
22:57:13 [INFO] train episode 30: reward = -500.00, steps = 500
22:57:14 [INFO] train episode 31: reward = -500.00, steps = 500
22:57:15 [INFO] train episode 32: reward = -500.00, steps = 500
22:57:16 [INFO] train episode 33: reward = -500.00, steps = 500
22:57:17 [INFO] train episode 34: reward = -500.00, steps = 500
22:57:18 [INFO] train episode 35: reward = -500.00, steps = 500
22:57:20 [INFO] train episode 36: reward = -500.00, steps = 500
22:57:21 [INFO] train episode 37: reward = -500.00, steps = 500
22:57:22 [INFO] train episode 38: reward = -500.00, steps = 500
22:57:23 [INFO] train episode 39: reward = -500.00, steps = 500
22:57:24 [INFO] train episode 40: reward = -500.00, steps = 500
22:57:25 [INFO] train episode 41: reward = -329.00, steps = 330
22:57:26 [INFO] train episode 42: reward = -500.00, steps = 500
22:57:28 [INFO] train episode 43: reward = -500.00, steps = 500
22:57:29 [INFO] train episode 44: reward = -500.00, steps = 500
22:57:30 [INFO] train episode 45: reward = -500.00, steps = 500
22:57:31 [INFO] train episode 46: reward = -500.00, steps = 500
22:57:32 [INFO] train episode 47: reward = -500.00, steps = 500
22:57:34 [INFO] train episode 48: reward = -500.00, steps = 500
22:57:35 [INFO] train episode 49: reward = -500.00, steps = 500
22:57:36 [INFO] train episode 50: reward = -467.00, steps = 468
22:57:37 [INFO] train episode 51: reward = -500.00, steps = 500
22:57:38 [INFO] train episode 52: reward = -491.00, steps = 492
22:57:39 [INFO] train episode 53: reward = -500.00, steps = 500
22:57:40 [INFO] train episode 54: reward = -500.00, steps = 500
22:57:41 [INFO] train episode 55: reward = -351.00, steps = 352
22:57:42 [INFO] train episode 56: reward = -500.00, steps = 500
22:57:43 [INFO] train episode 57: reward = -438.00, steps = 439
22:57:43 [INFO] train episode 58: reward = -303.00, steps = 304
22:57:44 [INFO] train episode 59: reward = -373.00, steps = 374
22:57:45 [INFO] train episode 60: reward = -328.00, steps = 329
22:57:45 [INFO] train episode 61: reward = -288.00, steps = 289
22:57:46 [INFO] train episode 62: reward = -332.00, steps = 333
22:57:47 [INFO] train episode 63: reward = -457.00, steps = 458
22:57:48 [INFO] train episode 64: reward = -500.00, steps = 500
22:57:49 [INFO] train episode 65: reward = -368.00, steps = 369
22:57:50 [INFO] train episode 66: reward = -500.00, steps = 500
22:57:50 [INFO] train episode 67: reward = -322.00, steps = 323
22:57:51 [INFO] train episode 68: reward = -278.00, steps = 279
22:57:51 [INFO] train episode 69: reward = -337.00, steps = 338
22:57:52 [INFO] train episode 70: reward = -359.00, steps = 360
22:57:53 [INFO] train episode 71: reward = -409.00, steps = 410
22:57:53 [INFO] train episode 72: reward = -283.00, steps = 284
22:57:54 [INFO] train episode 73: reward = -229.00, steps = 230
22:57:55 [INFO] train episode 74: reward = -241.00, steps = 242
22:57:56 [INFO] train episode 75: reward = -363.00, steps = 364
22:57:56 [INFO] train episode 76: reward = -321.00, steps = 322
22:57:57 [INFO] train episode 77: reward = -415.00, steps = 416
22:57:58 [INFO] train episode 78: reward = -500.00, steps = 500
22:57:59 [INFO] train episode 79: reward = -324.00, steps = 325
22:58:00 [INFO] train episode 80: reward = -331.00, steps = 332
22:58:00 [INFO] train episode 81: reward = -218.00, steps = 219
22:58:01 [INFO] train episode 82: reward = -325.00, steps = 326
22:58:02 [INFO] train episode 83: reward = -453.00, steps = 454
22:58:02 [INFO] train episode 84: reward = -212.00, steps = 213
22:58:02 [INFO] train episode 85: reward = -246.00, steps = 247
22:58:03 [INFO] train episode 86: reward = -358.00, steps = 359
22:58:03 [INFO] train episode 87: reward = -145.00, steps = 146
22:58:04 [INFO] train episode 88: reward = -118.00, steps = 119
22:58:04 [INFO] train episode 89: reward = -157.00, steps = 158
22:58:05 [INFO] train episode 90: reward = -228.00, steps = 229
22:58:05 [INFO] train episode 91: reward = -225.00, steps = 226
22:58:05 [INFO] train episode 92: reward = -275.00, steps = 276
22:58:06 [INFO] train episode 93: reward = -254.00, steps = 255
22:58:06 [INFO] train episode 94: reward = -235.00, steps = 236
22:58:07 [INFO] train episode 95: reward = -165.00, steps = 166
22:58:07 [INFO] train episode 96: reward = -199.00, steps = 200
22:58:07 [INFO] train episode 97: reward = -267.00, steps = 268
22:58:08 [INFO] train episode 98: reward = -226.00, steps = 227
22:58:08 [INFO] train episode 99: reward = -168.00, steps = 169
22:58:09 [INFO] train episode 100: reward = -329.00, steps = 330
22:58:09 [INFO] train episode 101: reward = -181.00, steps = 182
22:58:09 [INFO] train episode 102: reward = -194.00, steps = 195
22:58:10 [INFO] train episode 103: reward = -172.00, steps = 173
22:58:10 [INFO] train episode 104: reward = -223.00, steps = 224
22:58:11 [INFO] train episode 105: reward = -252.00, steps = 253
22:58:11 [INFO] train episode 106: reward = -205.00, steps = 206
22:58:11 [INFO] train episode 107: reward = -182.00, steps = 183
22:58:12 [INFO] train episode 108: reward = -180.00, steps = 181
22:58:12 [INFO] train episode 109: reward = -219.00, steps = 220
22:58:13 [INFO] train episode 110: reward = -195.00, steps = 196
22:58:13 [INFO] train episode 111: reward = -213.00, steps = 214
22:58:13 [INFO] train episode 112: reward = -190.00, steps = 191
22:58:14 [INFO] train episode 113: reward = -212.00, steps = 213
22:58:14 [INFO] train episode 114: reward = -212.00, steps = 213
22:58:14 [INFO] train episode 115: reward = -221.00, steps = 222
22:58:14 [INFO] train episode 116: reward = -175.00, steps = 176
22:58:15 [INFO] train episode 117: reward = -241.00, steps = 242
22:58:15 [INFO] train episode 118: reward = -163.00, steps = 164
22:58:15 [INFO] train episode 119: reward = -219.00, steps = 220
22:58:16 [INFO] train episode 120: reward = -222.00, steps = 223
22:58:16 [INFO] train episode 121: reward = -253.00, steps = 254
22:58:17 [INFO] train episode 122: reward = -223.00, steps = 224
22:58:17 [INFO] train episode 123: reward = -210.00, steps = 211
22:58:17 [INFO] train episode 124: reward = -224.00, steps = 225
22:58:18 [INFO] train episode 125: reward = -165.00, steps = 166
22:58:18 [INFO] train episode 126: reward = -268.00, steps = 269
22:58:18 [INFO] train episode 127: reward = -272.00, steps = 273
22:58:19 [INFO] train episode 128: reward = -207.00, steps = 208
22:58:19 [INFO] train episode 129: reward = -218.00, steps = 219
22:58:19 [INFO] train episode 130: reward = -188.00, steps = 189
22:58:20 [INFO] train episode 131: reward = -187.00, steps = 188
22:58:20 [INFO] train episode 132: reward = -301.00, steps = 302
22:58:21 [INFO] train episode 133: reward = -236.00, steps = 237
22:58:21 [INFO] train episode 134: reward = -368.00, steps = 369
22:58:21 [INFO] train episode 135: reward = -199.00, steps = 200
22:58:22 [INFO] train episode 136: reward = -394.00, steps = 395
22:58:22 [INFO] train episode 137: reward = -211.00, steps = 212
22:58:23 [INFO] train episode 138: reward = -182.00, steps = 183
22:58:23 [INFO] train episode 139: reward = -257.00, steps = 258
22:58:23 [INFO] train episode 140: reward = -163.00, steps = 164
22:58:23 [INFO] train episode 141: reward = -206.00, steps = 207
22:58:24 [INFO] train episode 142: reward = -294.00, steps = 295
22:58:24 [INFO] train episode 143: reward = -276.00, steps = 277
22:58:24 [INFO] train episode 144: reward = -163.00, steps = 164
22:58:25 [INFO] train episode 145: reward = -194.00, steps = 195
22:58:25 [INFO] train episode 146: reward = -231.00, steps = 232
22:58:25 [INFO] train episode 147: reward = -281.00, steps = 282
22:58:26 [INFO] train episode 148: reward = -135.00, steps = 136
22:58:26 [INFO] train episode 149: reward = -175.00, steps = 176
22:58:26 [INFO] train episode 150: reward = -264.00, steps = 265
22:58:27 [INFO] train episode 151: reward = -203.00, steps = 204
22:58:27 [INFO] train episode 152: reward = -186.00, steps = 187
22:58:27 [INFO] train episode 153: reward = -349.00, steps = 350
22:58:28 [INFO] train episode 154: reward = -307.00, steps = 308
22:58:28 [INFO] train episode 155: reward = -261.00, steps = 262
22:58:29 [INFO] train episode 156: reward = -278.00, steps = 279
22:58:29 [INFO] train episode 157: reward = -293.00, steps = 294
22:58:30 [INFO] train episode 158: reward = -251.00, steps = 252
22:58:30 [INFO] train episode 159: reward = -236.00, steps = 237
22:58:30 [INFO] train episode 160: reward = -281.00, steps = 282
22:58:31 [INFO] train episode 161: reward = -248.00, steps = 249
22:58:31 [INFO] train episode 162: reward = -206.00, steps = 207
22:58:31 [INFO] train episode 163: reward = -272.00, steps = 273
22:58:32 [INFO] train episode 164: reward = -274.00, steps = 275
22:58:32 [INFO] train episode 165: reward = -180.00, steps = 181
22:58:33 [INFO] train episode 166: reward = -500.00, steps = 500
22:58:33 [INFO] train episode 167: reward = -282.00, steps = 283
22:58:34 [INFO] train episode 168: reward = -243.00, steps = 244
22:58:34 [INFO] train episode 169: reward = -175.00, steps = 176
22:58:34 [INFO] train episode 170: reward = -279.00, steps = 280
22:58:35 [INFO] train episode 171: reward = -198.00, steps = 199
22:58:35 [INFO] train episode 172: reward = -195.00, steps = 196
22:58:35 [INFO] train episode 173: reward = -218.00, steps = 219
22:58:36 [INFO] train episode 174: reward = -328.00, steps = 329
22:58:36 [INFO] train episode 175: reward = -179.00, steps = 180
22:58:36 [INFO] train episode 176: reward = -197.00, steps = 198
22:58:37 [INFO] train episode 177: reward = -299.00, steps = 300
22:58:37 [INFO] train episode 178: reward = -279.00, steps = 280
22:58:38 [INFO] train episode 179: reward = -355.00, steps = 356
22:58:39 [INFO] train episode 180: reward = -325.00, steps = 326
22:58:39 [INFO] train episode 181: reward = -186.00, steps = 187
22:58:39 [INFO] train episode 182: reward = -250.00, steps = 251
22:58:40 [INFO] train episode 183: reward = -286.00, steps = 287
22:58:40 [INFO] train episode 184: reward = -177.00, steps = 178
22:58:41 [INFO] train episode 185: reward = -217.00, steps = 218
22:58:41 [INFO] train episode 186: reward = -223.00, steps = 224
22:58:41 [INFO] train episode 187: reward = -209.00, steps = 210
22:58:42 [INFO] train episode 188: reward = -210.00, steps = 211
22:58:42 [INFO] train episode 189: reward = -237.00, steps = 238
22:58:42 [INFO] train episode 190: reward = -182.00, steps = 183
22:58:43 [INFO] train episode 191: reward = -165.00, steps = 166
22:58:43 [INFO] train episode 192: reward = -204.00, steps = 205
22:58:43 [INFO] train episode 193: reward = -211.00, steps = 212
22:58:44 [INFO] train episode 194: reward = -174.00, steps = 175
22:58:44 [INFO] train episode 195: reward = -183.00, steps = 184
22:58:44 [INFO] train episode 196: reward = -272.00, steps = 273
22:58:45 [INFO] train episode 197: reward = -241.00, steps = 242
22:58:45 [INFO] train episode 198: reward = -257.00, steps = 258
22:58:45 [INFO] train episode 199: reward = -220.00, steps = 221
22:58:46 [INFO] train episode 200: reward = -226.00, steps = 227
22:58:46 [INFO] train episode 201: reward = -235.00, steps = 236
22:58:47 [INFO] train episode 202: reward = -257.00, steps = 258
22:58:47 [INFO] train episode 203: reward = -340.00, steps = 341
22:58:47 [INFO] train episode 204: reward = -202.00, steps = 203
22:58:48 [INFO] train episode 205: reward = -451.00, steps = 452
22:58:48 [INFO] train episode 206: reward = -231.00, steps = 232
22:58:49 [INFO] train episode 207: reward = -192.00, steps = 193
22:58:49 [INFO] train episode 208: reward = -245.00, steps = 246
22:58:49 [INFO] train episode 209: reward = -192.00, steps = 193
22:58:50 [INFO] train episode 210: reward = -231.00, steps = 232
22:58:50 [INFO] train episode 211: reward = -303.00, steps = 304
22:58:50 [INFO] train episode 212: reward = -223.00, steps = 224
22:58:51 [INFO] train episode 213: reward = -190.00, steps = 191
22:58:51 [INFO] train episode 214: reward = -188.00, steps = 189
22:58:51 [INFO] train episode 215: reward = -152.00, steps = 153
22:58:51 [INFO] train episode 216: reward = -157.00, steps = 158
22:58:52 [INFO] train episode 217: reward = -350.00, steps = 351
22:58:52 [INFO] train episode 218: reward = -124.00, steps = 125
22:58:53 [INFO] train episode 219: reward = -349.00, steps = 350
22:58:53 [INFO] train episode 220: reward = -171.00, steps = 172
22:58:53 [INFO] train episode 221: reward = -190.00, steps = 191
22:58:54 [INFO] train episode 222: reward = -325.00, steps = 326
22:58:54 [INFO] train episode 223: reward = -348.00, steps = 349
22:58:54 [INFO] train episode 224: reward = -235.00, steps = 236
22:58:55 [INFO] train episode 225: reward = -180.00, steps = 181
22:58:55 [INFO] train episode 226: reward = -203.00, steps = 204
22:58:55 [INFO] train episode 227: reward = -266.00, steps = 267
22:58:56 [INFO] train episode 228: reward = -217.00, steps = 218
22:58:56 [INFO] train episode 229: reward = -175.00, steps = 176
22:58:56 [INFO] train episode 230: reward = -160.00, steps = 161
22:58:56 [INFO] train episode 231: reward = -134.00, steps = 135
22:58:57 [INFO] train episode 232: reward = -152.00, steps = 153
22:58:57 [INFO] train episode 233: reward = -210.00, steps = 211
22:58:57 [INFO] train episode 234: reward = -216.00, steps = 217
22:58:57 [INFO] train episode 235: reward = -199.00, steps = 200
22:58:58 [INFO] train episode 236: reward = -122.00, steps = 123
22:58:58 [INFO] train episode 237: reward = -132.00, steps = 133
22:58:58 [INFO] train episode 238: reward = -310.00, steps = 311
22:58:58 [INFO] train episode 239: reward = -262.00, steps = 263
22:58:59 [INFO] train episode 240: reward = -196.00, steps = 197
22:58:59 [INFO] train episode 241: reward = -215.00, steps = 216
22:58:59 [INFO] train episode 242: reward = -160.00, steps = 161
22:59:00 [INFO] train episode 243: reward = -174.00, steps = 175
22:59:00 [INFO] train episode 244: reward = -282.00, steps = 283
22:59:00 [INFO] train episode 245: reward = -287.00, steps = 288
22:59:01 [INFO] train episode 246: reward = -222.00, steps = 223
22:59:01 [INFO] train episode 247: reward = -200.00, steps = 201
22:59:01 [INFO] train episode 248: reward = -240.00, steps = 241
22:59:01 [INFO] train episode 249: reward = -259.00, steps = 260
22:59:02 [INFO] train episode 250: reward = -251.00, steps = 252
22:59:02 [INFO] train episode 251: reward = -178.00, steps = 179
22:59:02 [INFO] train episode 252: reward = -176.00, steps = 177
22:59:02 [INFO] train episode 253: reward = -224.00, steps = 225
22:59:03 [INFO] train episode 254: reward = -136.00, steps = 137
22:59:03 [INFO] train episode 255: reward = -159.00, steps = 160
22:59:03 [INFO] train episode 256: reward = -166.00, steps = 167
22:59:03 [INFO] train episode 257: reward = -132.00, steps = 133
22:59:04 [INFO] train episode 258: reward = -251.00, steps = 252
22:59:04 [INFO] train episode 259: reward = -254.00, steps = 255
22:59:04 [INFO] train episode 260: reward = -151.00, steps = 152
22:59:04 [INFO] train episode 261: reward = -145.00, steps = 146
22:59:05 [INFO] train episode 262: reward = -214.00, steps = 215
22:59:05 [INFO] train episode 263: reward = -206.00, steps = 207
22:59:05 [INFO] train episode 264: reward = -174.00, steps = 175
22:59:05 [INFO] train episode 265: reward = -166.00, steps = 167
22:59:06 [INFO] train episode 266: reward = -273.00, steps = 274
22:59:06 [INFO] train episode 267: reward = -244.00, steps = 245
22:59:06 [INFO] train episode 268: reward = -285.00, steps = 286
22:59:06 [INFO] train episode 269: reward = -214.00, steps = 215
22:59:07 [INFO] train episode 270: reward = -168.00, steps = 169
22:59:07 [INFO] train episode 271: reward = -196.00, steps = 197
22:59:07 [INFO] train episode 272: reward = -213.00, steps = 214
22:59:07 [INFO] train episode 273: reward = -191.00, steps = 192
22:59:08 [INFO] train episode 274: reward = -193.00, steps = 194
22:59:08 [INFO] train episode 275: reward = -230.00, steps = 231
22:59:08 [INFO] train episode 276: reward = -260.00, steps = 261
22:59:09 [INFO] train episode 277: reward = -276.00, steps = 277
22:59:09 [INFO] train episode 278: reward = -189.00, steps = 190
22:59:09 [INFO] train episode 279: reward = -246.00, steps = 247
22:59:10 [INFO] train episode 280: reward = -226.00, steps = 227
22:59:10 [INFO] train episode 281: reward = -215.00, steps = 216
22:59:10 [INFO] train episode 282: reward = -241.00, steps = 242
22:59:10 [INFO] train episode 283: reward = -189.00, steps = 190
22:59:11 [INFO] train episode 284: reward = -205.00, steps = 206
22:59:11 [INFO] train episode 285: reward = -248.00, steps = 249
22:59:11 [INFO] train episode 286: reward = -321.00, steps = 322
22:59:12 [INFO] train episode 287: reward = -222.00, steps = 223
22:59:12 [INFO] train episode 288: reward = -208.00, steps = 209
22:59:12 [INFO] train episode 289: reward = -254.00, steps = 255
22:59:12 [INFO] train episode 290: reward = -188.00, steps = 189
22:59:12 [INFO] train episode 291: reward = -189.00, steps = 190
22:59:13 [INFO] train episode 292: reward = -220.00, steps = 221
22:59:13 [INFO] train episode 293: reward = -241.00, steps = 242
22:59:13 [INFO] train episode 294: reward = -231.00, steps = 232
22:59:14 [INFO] train episode 295: reward = -220.00, steps = 221
22:59:14 [INFO] train episode 296: reward = -189.00, steps = 190
22:59:14 [INFO] train episode 297: reward = -147.00, steps = 148
22:59:14 [INFO] train episode 298: reward = -244.00, steps = 245
22:59:15 [INFO] train episode 299: reward = -144.00, steps = 145
22:59:15 [INFO] train episode 300: reward = -210.00, steps = 211
22:59:15 [INFO] train episode 301: reward = -173.00, steps = 174
22:59:15 [INFO] train episode 302: reward = -208.00, steps = 209
22:59:16 [INFO] train episode 303: reward = -141.00, steps = 142
22:59:16 [INFO] train episode 304: reward = -250.00, steps = 251
22:59:16 [INFO] train episode 305: reward = -204.00, steps = 205
22:59:16 [INFO] train episode 306: reward = -175.00, steps = 176
22:59:17 [INFO] train episode 307: reward = -199.00, steps = 200
22:59:17 [INFO] train episode 308: reward = -191.00, steps = 192
22:59:17 [INFO] train episode 309: reward = -249.00, steps = 250
22:59:17 [INFO] train episode 310: reward = -168.00, steps = 169
22:59:18 [INFO] train episode 311: reward = -180.00, steps = 181
22:59:18 [INFO] train episode 312: reward = -173.00, steps = 174
22:59:18 [INFO] train episode 313: reward = -260.00, steps = 261
22:59:19 [INFO] train episode 314: reward = -201.00, steps = 202
22:59:19 [INFO] train episode 315: reward = -161.00, steps = 162
22:59:19 [INFO] train episode 316: reward = -161.00, steps = 162
22:59:19 [INFO] train episode 317: reward = -200.00, steps = 201
22:59:20 [INFO] train episode 318: reward = -171.00, steps = 172
22:59:20 [INFO] train episode 319: reward = -191.00, steps = 192
22:59:20 [INFO] train episode 320: reward = -264.00, steps = 265
22:59:20 [INFO] train episode 321: reward = -175.00, steps = 176
22:59:21 [INFO] train episode 322: reward = -152.00, steps = 153
22:59:21 [INFO] train episode 323: reward = -168.00, steps = 169
22:59:21 [INFO] train episode 324: reward = -235.00, steps = 236
22:59:21 [INFO] train episode 325: reward = -222.00, steps = 223
22:59:22 [INFO] train episode 326: reward = -191.00, steps = 192
22:59:22 [INFO] train episode 327: reward = -185.00, steps = 186
22:59:22 [INFO] train episode 328: reward = -205.00, steps = 206
22:59:22 [INFO] train episode 329: reward = -175.00, steps = 176
22:59:23 [INFO] train episode 330: reward = -190.00, steps = 191
22:59:23 [INFO] train episode 331: reward = -199.00, steps = 200
22:59:23 [INFO] train episode 332: reward = -155.00, steps = 156
22:59:23 [INFO] train episode 333: reward = -218.00, steps = 219
22:59:24 [INFO] train episode 334: reward = -170.00, steps = 171
22:59:24 [INFO] train episode 335: reward = -223.00, steps = 224
22:59:24 [INFO] train episode 336: reward = -184.00, steps = 185
22:59:25 [INFO] train episode 337: reward = -363.00, steps = 364
22:59:25 [INFO] train episode 338: reward = -218.00, steps = 219
22:59:25 [INFO] train episode 339: reward = -162.00, steps = 163
22:59:25 [INFO] train episode 340: reward = -167.00, steps = 168
22:59:26 [INFO] train episode 341: reward = -200.00, steps = 201
22:59:26 [INFO] train episode 342: reward = -295.00, steps = 296
22:59:26 [INFO] train episode 343: reward = -128.00, steps = 129
22:59:26 [INFO] train episode 344: reward = -193.00, steps = 194
22:59:27 [INFO] train episode 345: reward = -221.00, steps = 222
22:59:27 [INFO] train episode 346: reward = -162.00, steps = 163
22:59:27 [INFO] train episode 347: reward = -207.00, steps = 208
22:59:27 [INFO] train episode 348: reward = -164.00, steps = 165
22:59:28 [INFO] train episode 349: reward = -163.00, steps = 164
22:59:28 [INFO] train episode 350: reward = -197.00, steps = 198
22:59:28 [INFO] train episode 351: reward = -162.00, steps = 163
22:59:28 [INFO] train episode 352: reward = -208.00, steps = 209
22:59:29 [INFO] train episode 353: reward = -183.00, steps = 184
22:59:29 [INFO] train episode 354: reward = -179.00, steps = 180
22:59:29 [INFO] train episode 355: reward = -285.00, steps = 286
22:59:29 [INFO] train episode 356: reward = -158.00, steps = 159
22:59:30 [INFO] train episode 357: reward = -194.00, steps = 195
22:59:30 [INFO] train episode 358: reward = -171.00, steps = 172
22:59:30 [INFO] train episode 359: reward = -177.00, steps = 178
22:59:30 [INFO] train episode 360: reward = -194.00, steps = 195
22:59:31 [INFO] train episode 361: reward = -226.00, steps = 227
22:59:31 [INFO] train episode 362: reward = -185.00, steps = 186
22:59:31 [INFO] train episode 363: reward = -191.00, steps = 192
22:59:31 [INFO] train episode 364: reward = -165.00, steps = 166
22:59:32 [INFO] train episode 365: reward = -162.00, steps = 163
22:59:32 [INFO] train episode 366: reward = -190.00, steps = 191
22:59:32 [INFO] train episode 367: reward = -161.00, steps = 162
22:59:32 [INFO] train episode 368: reward = -209.00, steps = 210
22:59:33 [INFO] train episode 369: reward = -422.00, steps = 423
22:59:33 [INFO] train episode 370: reward = -149.00, steps = 150
22:59:33 [INFO] train episode 371: reward = -195.00, steps = 196
22:59:33 [INFO] train episode 372: reward = -192.00, steps = 193
22:59:34 [INFO] train episode 373: reward = -153.00, steps = 154
22:59:34 [INFO] train episode 374: reward = -181.00, steps = 182
22:59:34 [INFO] train episode 375: reward = -155.00, steps = 156
22:59:34 [INFO] train episode 376: reward = -229.00, steps = 230
22:59:35 [INFO] train episode 377: reward = -178.00, steps = 179
22:59:35 [INFO] train episode 378: reward = -188.00, steps = 189
22:59:35 [INFO] train episode 379: reward = -150.00, steps = 151
22:59:35 [INFO] train episode 380: reward = -282.00, steps = 283
22:59:36 [INFO] train episode 381: reward = -160.00, steps = 161
22:59:36 [INFO] train episode 382: reward = -229.00, steps = 230
22:59:36 [INFO] train episode 383: reward = -177.00, steps = 178
22:59:36 [INFO] train episode 384: reward = -246.00, steps = 247
22:59:37 [INFO] train episode 385: reward = -215.00, steps = 216
22:59:37 [INFO] train episode 386: reward = -170.00, steps = 171
22:59:37 [INFO] train episode 387: reward = -195.00, steps = 196
22:59:37 [INFO] train episode 388: reward = -192.00, steps = 193
22:59:38 [INFO] train episode 389: reward = -168.00, steps = 169
22:59:38 [INFO] train episode 390: reward = -166.00, steps = 167
22:59:38 [INFO] train episode 391: reward = -187.00, steps = 188
22:59:38 [INFO] train episode 392: reward = -178.00, steps = 179
22:59:39 [INFO] train episode 393: reward = -183.00, steps = 184
22:59:39 [INFO] train episode 394: reward = -186.00, steps = 187
22:59:39 [INFO] train episode 395: reward = -221.00, steps = 222
22:59:39 [INFO] train episode 396: reward = -159.00, steps = 160
22:59:40 [INFO] train episode 397: reward = -133.00, steps = 134
22:59:40 [INFO] train episode 398: reward = -257.00, steps = 258
22:59:40 [INFO] train episode 399: reward = -266.00, steps = 267
22:59:40 [INFO] train episode 400: reward = -195.00, steps = 196
22:59:41 [INFO] train episode 401: reward = -173.00, steps = 174
22:59:41 [INFO] train episode 402: reward = -144.00, steps = 145
22:59:41 [INFO] train episode 403: reward = -174.00, steps = 175
22:59:41 [INFO] train episode 404: reward = -192.00, steps = 193
22:59:42 [INFO] train episode 405: reward = -178.00, steps = 179
22:59:42 [INFO] train episode 406: reward = -228.00, steps = 229
22:59:42 [INFO] train episode 407: reward = -207.00, steps = 208
22:59:42 [INFO] train episode 408: reward = -156.00, steps = 157
22:59:43 [INFO] train episode 409: reward = -216.00, steps = 217
22:59:43 [INFO] train episode 410: reward = -135.00, steps = 136
22:59:43 [INFO] train episode 411: reward = -194.00, steps = 195
22:59:43 [INFO] train episode 412: reward = -186.00, steps = 187
22:59:44 [INFO] train episode 413: reward = -253.00, steps = 254
22:59:44 [INFO] train episode 414: reward = -268.00, steps = 269
22:59:44 [INFO] train episode 415: reward = -173.00, steps = 174
22:59:45 [INFO] train episode 416: reward = -351.00, steps = 352
22:59:45 [INFO] train episode 417: reward = -204.00, steps = 205
22:59:45 [INFO] train episode 418: reward = -180.00, steps = 181
22:59:46 [INFO] train episode 419: reward = -198.00, steps = 199
22:59:46 [INFO] train episode 420: reward = -206.00, steps = 207
22:59:46 [INFO] train episode 421: reward = -146.00, steps = 147
22:59:46 [INFO] train episode 422: reward = -191.00, steps = 192
22:59:47 [INFO] train episode 423: reward = -222.00, steps = 223
22:59:47 [INFO] train episode 424: reward = -197.00, steps = 198
22:59:47 [INFO] train episode 425: reward = -138.00, steps = 139
22:59:47 [INFO] train episode 426: reward = -209.00, steps = 210
22:59:48 [INFO] train episode 427: reward = -189.00, steps = 190
22:59:48 [INFO] train episode 428: reward = -181.00, steps = 182
22:59:48 [INFO] train episode 429: reward = -165.00, steps = 166
22:59:48 [INFO] train episode 430: reward = -186.00, steps = 187
22:59:49 [INFO] train episode 431: reward = -171.00, steps = 172
22:59:49 [INFO] train episode 432: reward = -175.00, steps = 176
22:59:49 [INFO] train episode 433: reward = -216.00, steps = 217
22:59:49 [INFO] train episode 434: reward = -143.00, steps = 144
22:59:50 [INFO] train episode 435: reward = -151.00, steps = 152
22:59:50 [INFO] train episode 436: reward = -132.00, steps = 133
22:59:50 [INFO] train episode 437: reward = -151.00, steps = 152
22:59:50 [INFO] train episode 438: reward = -232.00, steps = 233
22:59:50 [INFO] train episode 439: reward = -166.00, steps = 167
22:59:51 [INFO] train episode 440: reward = -179.00, steps = 180
22:59:51 [INFO] train episode 441: reward = -166.00, steps = 167
22:59:51 [INFO] train episode 442: reward = -193.00, steps = 194
22:59:51 [INFO] train episode 443: reward = -152.00, steps = 153
22:59:52 [INFO] train episode 444: reward = -291.00, steps = 292
22:59:52 [INFO] train episode 445: reward = -144.00, steps = 145
22:59:52 [INFO] train episode 446: reward = -261.00, steps = 262
22:59:53 [INFO] train episode 447: reward = -178.00, steps = 179
22:59:53 [INFO] train episode 448: reward = -242.00, steps = 243
22:59:53 [INFO] train episode 449: reward = -155.00, steps = 156
22:59:53 [INFO] train episode 450: reward = -188.00, steps = 189
22:59:54 [INFO] train episode 451: reward = -159.00, steps = 160
22:59:54 [INFO] train episode 452: reward = -311.00, steps = 312
22:59:54 [INFO] train episode 453: reward = -188.00, steps = 189
22:59:54 [INFO] train episode 454: reward = -270.00, steps = 271
22:59:55 [INFO] train episode 455: reward = -224.00, steps = 225
22:59:55 [INFO] train episode 456: reward = -122.00, steps = 123
22:59:55 [INFO] train episode 457: reward = -210.00, steps = 211
22:59:55 [INFO] train episode 458: reward = -156.00, steps = 157
22:59:55 [INFO] train episode 459: reward = -147.00, steps = 148
22:59:55 [INFO] train episode 460: reward = -143.00, steps = 144
22:59:56 [INFO] train episode 461: reward = -256.00, steps = 257
22:59:56 [INFO] train episode 462: reward = -158.00, steps = 159
22:59:56 [INFO] train episode 463: reward = -196.00, steps = 197
22:59:56 [INFO] train episode 464: reward = -175.00, steps = 176
22:59:57 [INFO] train episode 465: reward = -196.00, steps = 197
22:59:57 [INFO] train episode 466: reward = -159.00, steps = 160
22:59:57 [INFO] train episode 467: reward = -143.00, steps = 144
22:59:57 [INFO] train episode 468: reward = -192.00, steps = 193
22:59:57 [INFO] train episode 469: reward = -204.00, steps = 205
22:59:58 [INFO] train episode 470: reward = -229.00, steps = 230
22:59:58 [INFO] train episode 471: reward = -153.00, steps = 154
22:59:58 [INFO] train episode 472: reward = -304.00, steps = 305
22:59:58 [INFO] train episode 473: reward = -149.00, steps = 150
22:59:59 [INFO] train episode 474: reward = -162.00, steps = 163
22:59:59 [INFO] train episode 475: reward = -170.00, steps = 171
22:59:59 [INFO] train episode 476: reward = -117.00, steps = 118
22:59:59 [INFO] train episode 477: reward = -153.00, steps = 154
22:59:59 [INFO] train episode 478: reward = -190.00, steps = 191
23:00:00 [INFO] train episode 479: reward = -223.00, steps = 224
23:00:00 [INFO] train episode 480: reward = -183.00, steps = 184
23:00:00 [INFO] train episode 481: reward = -149.00, steps = 150
23:00:00 [INFO] train episode 482: reward = -208.00, steps = 209
23:00:00 [INFO] train episode 483: reward = -176.00, steps = 177
23:00:01 [INFO] train episode 484: reward = -179.00, steps = 180
23:00:01 [INFO] train episode 485: reward = -153.00, steps = 154
23:00:01 [INFO] train episode 486: reward = -200.00, steps = 201
23:00:01 [INFO] train episode 487: reward = -179.00, steps = 180
23:00:01 [INFO] train episode 488: reward = -191.00, steps = 192
23:00:02 [INFO] train episode 489: reward = -167.00, steps = 168
23:00:02 [INFO] train episode 490: reward = -184.00, steps = 185
23:00:02 [INFO] train episode 491: reward = -173.00, steps = 174
23:00:02 [INFO] train episode 492: reward = -195.00, steps = 196
23:00:03 [INFO] train episode 493: reward = -268.00, steps = 269
23:00:03 [INFO] train episode 494: reward = -160.00, steps = 161
23:00:03 [INFO] train episode 495: reward = -181.00, steps = 182
23:00:03 [INFO] train episode 496: reward = -155.00, steps = 156
23:00:03 [INFO] train episode 497: reward = -143.00, steps = 144
23:00:03 [INFO] train episode 498: reward = -166.00, steps = 167
23:00:04 [INFO] train episode 499: reward = -207.00, steps = 208
23:00:04 [INFO] train episode 500: reward = -193.00, steps = 194
23:00:04 [INFO] train episode 501: reward = -194.00, steps = 195
23:00:04 [INFO] train episode 502: reward = -138.00, steps = 139
23:00:05 [INFO] train episode 503: reward = -249.00, steps = 250
23:00:05 [INFO] train episode 504: reward = -289.00, steps = 290
23:00:05 [INFO] train episode 505: reward = -241.00, steps = 242
23:00:05 [INFO] train episode 506: reward = -132.00, steps = 133
23:00:05 [INFO] train episode 507: reward = -150.00, steps = 151
23:00:06 [INFO] train episode 508: reward = -176.00, steps = 177
23:00:06 [INFO] train episode 509: reward = -225.00, steps = 226
23:00:06 [INFO] train episode 510: reward = -261.00, steps = 262
23:00:06 [INFO] train episode 511: reward = -155.00, steps = 156
23:00:07 [INFO] train episode 512: reward = -159.00, steps = 160
23:00:07 [INFO] train episode 513: reward = -172.00, steps = 173
23:00:07 [INFO] train episode 514: reward = -165.00, steps = 166
23:00:07 [INFO] train episode 515: reward = -144.00, steps = 145
23:00:07 [INFO] train episode 516: reward = -156.00, steps = 157
23:00:07 [INFO] train episode 517: reward = -184.00, steps = 185
23:00:08 [INFO] train episode 518: reward = -172.00, steps = 173
23:00:08 [INFO] train episode 519: reward = -296.00, steps = 297
23:00:08 [INFO] train episode 520: reward = -110.00, steps = 111
23:00:08 [INFO] train episode 521: reward = -159.00, steps = 160
23:00:08 [INFO] train episode 522: reward = -190.00, steps = 191
23:00:09 [INFO] train episode 523: reward = -262.00, steps = 263
23:00:09 [INFO] train episode 524: reward = -158.00, steps = 159
23:00:09 [INFO] train episode 525: reward = -151.00, steps = 152
23:00:09 [INFO] train episode 526: reward = -162.00, steps = 163
23:00:09 [INFO] train episode 527: reward = -176.00, steps = 177
23:00:10 [INFO] train episode 528: reward = -190.00, steps = 191
23:00:10 [INFO] train episode 529: reward = -175.00, steps = 176
23:00:10 [INFO] train episode 530: reward = -206.00, steps = 207
23:00:10 [INFO] train episode 531: reward = -244.00, steps = 245
23:00:10 [INFO] train episode 532: reward = -136.00, steps = 137
23:00:11 [INFO] train episode 533: reward = -141.00, steps = 142
23:00:11 [INFO] train episode 534: reward = -133.00, steps = 134
23:00:11 [INFO] train episode 535: reward = -206.00, steps = 207
23:00:11 [INFO] train episode 536: reward = -169.00, steps = 170
23:00:11 [INFO] train episode 537: reward = -178.00, steps = 179
23:00:11 [INFO] train episode 538: reward = -161.00, steps = 162
23:00:12 [INFO] train episode 539: reward = -139.00, steps = 140
23:00:12 [INFO] train episode 540: reward = -192.00, steps = 193
23:00:12 [INFO] train episode 541: reward = -200.00, steps = 201
23:00:12 [INFO] train episode 542: reward = -115.00, steps = 116
23:00:12 [INFO] train episode 543: reward = -163.00, steps = 164
23:00:13 [INFO] train episode 544: reward = -277.00, steps = 278
23:00:13 [INFO] train episode 545: reward = -190.00, steps = 191
23:00:13 [INFO] train episode 546: reward = -179.00, steps = 180
23:00:13 [INFO] train episode 547: reward = -187.00, steps = 188
23:00:14 [INFO] train episode 548: reward = -314.00, steps = 315
23:00:14 [INFO] train episode 549: reward = -187.00, steps = 188
23:00:14 [INFO] train episode 550: reward = -243.00, steps = 244
23:00:15 [INFO] train episode 551: reward = -174.00, steps = 175
23:00:15 [INFO] train episode 552: reward = -197.00, steps = 198
23:00:15 [INFO] train episode 553: reward = -191.00, steps = 192
23:00:15 [INFO] train episode 554: reward = -175.00, steps = 176
23:00:15 [INFO] train episode 555: reward = -184.00, steps = 185
23:00:15 [INFO] train episode 556: reward = -144.00, steps = 145
23:00:16 [INFO] train episode 557: reward = -205.00, steps = 206
23:00:16 [INFO] train episode 558: reward = -152.00, steps = 153
23:00:16 [INFO] train episode 559: reward = -165.00, steps = 166
23:00:16 [INFO] train episode 560: reward = -173.00, steps = 174
23:00:16 [INFO] train episode 561: reward = -184.00, steps = 185
23:00:17 [INFO] train episode 562: reward = -167.00, steps = 168
23:00:17 [INFO] train episode 563: reward = -221.00, steps = 222
23:00:17 [INFO] train episode 564: reward = -184.00, steps = 185
23:00:17 [INFO] train episode 565: reward = -188.00, steps = 189
23:00:18 [INFO] train episode 566: reward = -181.00, steps = 182
23:00:18 [INFO] train episode 567: reward = -170.00, steps = 171
23:00:18 [INFO] train episode 568: reward = -189.00, steps = 190
23:00:18 [INFO] train episode 569: reward = -159.00, steps = 160
23:00:18 [INFO] train episode 570: reward = -186.00, steps = 187
23:00:19 [INFO] train episode 571: reward = -168.00, steps = 169
23:00:19 [INFO] train episode 572: reward = -140.00, steps = 141
23:00:19 [INFO] train episode 573: reward = -194.00, steps = 195
23:00:19 [INFO] train episode 574: reward = -220.00, steps = 221
23:00:19 [INFO] train episode 575: reward = -168.00, steps = 169
23:00:20 [INFO] train episode 576: reward = -156.00, steps = 157
23:00:20 [INFO] train episode 577: reward = -196.00, steps = 197
23:00:20 [INFO] train episode 578: reward = -170.00, steps = 171
23:00:20 [INFO] train episode 579: reward = -181.00, steps = 182
23:00:21 [INFO] train episode 580: reward = -364.00, steps = 365
23:00:21 [INFO] train episode 581: reward = -215.00, steps = 216
23:00:21 [INFO] train episode 582: reward = -166.00, steps = 167
23:00:21 [INFO] train episode 583: reward = -183.00, steps = 184
23:00:21 [INFO] train episode 584: reward = -164.00, steps = 165
23:00:22 [INFO] train episode 585: reward = -131.00, steps = 132
23:00:22 [INFO] train episode 586: reward = -235.00, steps = 236
23:00:22 [INFO] train episode 587: reward = -202.00, steps = 203
23:00:22 [INFO] train episode 588: reward = -170.00, steps = 171
23:00:22 [INFO] train episode 589: reward = -151.00, steps = 152
23:00:23 [INFO] train episode 590: reward = -205.00, steps = 206
23:00:23 [INFO] train episode 591: reward = -168.00, steps = 169
23:00:23 [INFO] train episode 592: reward = -179.00, steps = 180
23:00:23 [INFO] train episode 593: reward = -216.00, steps = 217
23:00:23 [INFO] train episode 594: reward = -181.00, steps = 182
23:00:24 [INFO] train episode 595: reward = -149.00, steps = 150
23:00:24 [INFO] train episode 596: reward = -138.00, steps = 139
23:00:24 [INFO] train episode 597: reward = -160.00, steps = 161
23:00:24 [INFO] train episode 598: reward = -163.00, steps = 164
23:00:24 [INFO] train episode 599: reward = -179.00, steps = 180
23:00:25 [INFO] train episode 600: reward = -191.00, steps = 192
23:00:25 [INFO] train episode 601: reward = -202.00, steps = 203
23:00:25 [INFO] train episode 602: reward = -247.00, steps = 248
23:00:25 [INFO] train episode 603: reward = -252.00, steps = 253
23:00:25 [INFO] train episode 604: reward = -209.00, steps = 210
23:00:26 [INFO] train episode 605: reward = -168.00, steps = 169
23:00:26 [INFO] train episode 606: reward = -226.00, steps = 227
23:00:26 [INFO] train episode 607: reward = -153.00, steps = 154
23:00:26 [INFO] train episode 608: reward = -130.00, steps = 131
23:00:27 [INFO] train episode 609: reward = -184.00, steps = 185
23:00:27 [INFO] train episode 610: reward = -179.00, steps = 180
23:00:27 [INFO] train episode 611: reward = -165.00, steps = 166
23:00:27 [INFO] train episode 612: reward = -326.00, steps = 327
23:00:27 [INFO] train episode 613: reward = -200.00, steps = 201
23:00:28 [INFO] train episode 614: reward = -139.00, steps = 140
23:00:28 [INFO] train episode 615: reward = -230.00, steps = 231
23:00:28 [INFO] train episode 616: reward = -181.00, steps = 182
23:00:28 [INFO] train episode 617: reward = -190.00, steps = 191
23:00:28 [INFO] train episode 618: reward = -137.00, steps = 138
23:00:29 [INFO] train episode 619: reward = -183.00, steps = 184
23:00:29 [INFO] train episode 620: reward = -210.00, steps = 211
23:00:29 [INFO] train episode 621: reward = -151.00, steps = 152
23:00:29 [INFO] train episode 622: reward = -295.00, steps = 296
23:00:30 [INFO] train episode 623: reward = -227.00, steps = 228
23:00:30 [INFO] train episode 624: reward = -120.00, steps = 121
23:00:30 [INFO] train episode 625: reward = -201.00, steps = 202
23:00:30 [INFO] train episode 626: reward = -210.00, steps = 211
23:00:30 [INFO] train episode 627: reward = -140.00, steps = 141
23:00:30 [INFO] train episode 628: reward = -158.00, steps = 159
23:00:31 [INFO] train episode 629: reward = -137.00, steps = 138
23:00:31 [INFO] train episode 630: reward = -236.00, steps = 237
23:00:31 [INFO] train episode 631: reward = -172.00, steps = 173
23:00:31 [INFO] train episode 632: reward = -297.00, steps = 298
23:00:32 [INFO] train episode 633: reward = -126.00, steps = 127
23:00:32 [INFO] train episode 634: reward = -172.00, steps = 173
23:00:32 [INFO] train episode 635: reward = -253.00, steps = 254
23:00:32 [INFO] train episode 636: reward = -202.00, steps = 203
23:00:32 [INFO] train episode 637: reward = -203.00, steps = 204
23:00:33 [INFO] train episode 638: reward = -179.00, steps = 180
23:00:33 [INFO] train episode 639: reward = -169.00, steps = 170
23:00:33 [INFO] train episode 640: reward = -166.00, steps = 167
23:00:33 [INFO] train episode 641: reward = -167.00, steps = 168
23:00:34 [INFO] train episode 642: reward = -272.00, steps = 273
23:00:34 [INFO] train episode 643: reward = -165.00, steps = 166
23:00:34 [INFO] train episode 644: reward = -153.00, steps = 154
23:00:34 [INFO] train episode 645: reward = -165.00, steps = 166
23:00:34 [INFO] train episode 646: reward = -225.00, steps = 226
23:00:35 [INFO] train episode 647: reward = -172.00, steps = 173
23:00:35 [INFO] train episode 648: reward = -204.00, steps = 205
23:00:35 [INFO] train episode 649: reward = -142.00, steps = 143
23:00:35 [INFO] train episode 650: reward = -180.00, steps = 181
23:00:35 [INFO] train episode 651: reward = -194.00, steps = 195
23:00:36 [INFO] train episode 652: reward = -195.00, steps = 196
23:00:36 [INFO] train episode 653: reward = -184.00, steps = 185
23:00:36 [INFO] train episode 654: reward = -114.00, steps = 115
23:00:36 [INFO] train episode 655: reward = -167.00, steps = 168
23:00:36 [INFO] train episode 656: reward = -201.00, steps = 202
23:00:37 [INFO] train episode 657: reward = -154.00, steps = 155
23:00:37 [INFO] train episode 658: reward = -124.00, steps = 125
23:00:37 [INFO] train episode 659: reward = -167.00, steps = 168
23:00:37 [INFO] train episode 660: reward = -175.00, steps = 176
23:00:37 [INFO] train episode 661: reward = -229.00, steps = 230
23:00:37 [INFO] train episode 662: reward = -158.00, steps = 159
23:00:38 [INFO] train episode 663: reward = -196.00, steps = 197
23:00:38 [INFO] train episode 664: reward = -201.00, steps = 202
23:00:38 [INFO] train episode 665: reward = -211.00, steps = 212
23:00:38 [INFO] train episode 666: reward = -195.00, steps = 196
23:00:39 [INFO] train episode 667: reward = -196.00, steps = 197
23:00:39 [INFO] train episode 668: reward = -257.00, steps = 258
23:00:39 [INFO] train episode 669: reward = -135.00, steps = 136
23:00:39 [INFO] train episode 670: reward = -214.00, steps = 215
23:00:39 [INFO] train episode 671: reward = -176.00, steps = 177
23:00:40 [INFO] train episode 672: reward = -162.00, steps = 163
23:00:40 [INFO] train episode 673: reward = -175.00, steps = 176
23:00:40 [INFO] train episode 674: reward = -200.00, steps = 201
23:00:40 [INFO] train episode 675: reward = -191.00, steps = 192
23:00:40 [INFO] train episode 676: reward = -128.00, steps = 129
23:00:41 [INFO] train episode 677: reward = -143.00, steps = 144
23:00:41 [INFO] train episode 678: reward = -187.00, steps = 188
23:00:41 [INFO] train episode 679: reward = -129.00, steps = 130
23:00:41 [INFO] train episode 680: reward = -175.00, steps = 176
23:00:41 [INFO] train episode 681: reward = -171.00, steps = 172
23:00:42 [INFO] train episode 682: reward = -324.00, steps = 325
23:00:42 [INFO] train episode 683: reward = -131.00, steps = 132
23:00:42 [INFO] train episode 684: reward = -152.00, steps = 153
23:00:42 [INFO] train episode 685: reward = -181.00, steps = 182
23:00:42 [INFO] train episode 686: reward = -144.00, steps = 145
23:00:43 [INFO] train episode 687: reward = -200.00, steps = 201
23:00:43 [INFO] train episode 688: reward = -209.00, steps = 210
23:00:43 [INFO] train episode 689: reward = -152.00, steps = 153
23:00:43 [INFO] train episode 690: reward = -207.00, steps = 208
23:00:43 [INFO] train episode 691: reward = -130.00, steps = 131
23:00:44 [INFO] train episode 692: reward = -169.00, steps = 170
23:00:44 [INFO] train episode 693: reward = -180.00, steps = 181
23:00:44 [INFO] train episode 694: reward = -324.00, steps = 325
23:00:44 [INFO] train episode 695: reward = -175.00, steps = 176
23:00:44 [INFO] train episode 696: reward = -191.00, steps = 192
23:00:45 [INFO] train episode 697: reward = -201.00, steps = 202
23:00:45 [INFO] train episode 698: reward = -180.00, steps = 181
23:00:45 [INFO] train episode 699: reward = -172.00, steps = 173
23:00:45 [INFO] train episode 700: reward = -181.00, steps = 182
23:00:45 [INFO] train episode 701: reward = -140.00, steps = 141
23:00:46 [INFO] train episode 702: reward = -149.00, steps = 150
23:00:46 [INFO] train episode 703: reward = -291.00, steps = 292
23:00:46 [INFO] train episode 704: reward = -188.00, steps = 189
23:00:46 [INFO] train episode 705: reward = -210.00, steps = 211
23:00:46 [INFO] train episode 706: reward = -169.00, steps = 170
23:00:47 [INFO] train episode 707: reward = -167.00, steps = 168
23:00:47 [INFO] train episode 708: reward = -175.00, steps = 176
23:00:47 [INFO] train episode 709: reward = -145.00, steps = 146
23:00:47 [INFO] train episode 710: reward = -168.00, steps = 169
23:00:47 [INFO] train episode 711: reward = -142.00, steps = 143
23:00:48 [INFO] train episode 712: reward = -151.00, steps = 152
23:00:48 [INFO] train episode 713: reward = -118.00, steps = 119
23:00:48 [INFO] train episode 714: reward = -255.00, steps = 256
23:00:48 [INFO] train episode 715: reward = -249.00, steps = 250
23:00:48 [INFO] train episode 716: reward = -155.00, steps = 156
23:00:49 [INFO] train episode 717: reward = -163.00, steps = 164
23:00:49 [INFO] train episode 718: reward = -167.00, steps = 168
23:00:49 [INFO] train episode 719: reward = -163.00, steps = 164
23:00:49 [INFO] train episode 720: reward = -141.00, steps = 142
23:00:49 [INFO] train episode 721: reward = -148.00, steps = 149
23:00:50 [INFO] train episode 722: reward = -187.00, steps = 188
23:00:50 [INFO] train episode 723: reward = -177.00, steps = 178
23:00:50 [INFO] train episode 724: reward = -183.00, steps = 184
23:00:50 [INFO] train episode 725: reward = -221.00, steps = 222
23:00:51 [INFO] train episode 726: reward = -189.00, steps = 190
23:00:51 [INFO] train episode 727: reward = -167.00, steps = 168
23:00:51 [INFO] train episode 728: reward = -140.00, steps = 141
23:00:51 [INFO] train episode 729: reward = -181.00, steps = 182
23:00:51 [INFO] train episode 730: reward = -214.00, steps = 215
23:00:52 [INFO] train episode 731: reward = -192.00, steps = 193
23:00:52 [INFO] train episode 732: reward = -210.00, steps = 211
23:00:52 [INFO] train episode 733: reward = -155.00, steps = 156
23:00:52 [INFO] train episode 734: reward = -174.00, steps = 175
23:00:52 [INFO] train episode 735: reward = -163.00, steps = 164
23:00:53 [INFO] train episode 736: reward = -161.00, steps = 162
23:00:53 [INFO] train episode 737: reward = -156.00, steps = 157
23:00:53 [INFO] train episode 738: reward = -136.00, steps = 137
23:00:53 [INFO] train episode 739: reward = -197.00, steps = 198
23:00:53 [INFO] train episode 740: reward = -197.00, steps = 198
23:00:54 [INFO] train episode 741: reward = -131.00, steps = 132
23:00:54 [INFO] train episode 742: reward = -171.00, steps = 172
23:00:54 [INFO] train episode 743: reward = -117.00, steps = 118
23:00:54 [INFO] train episode 744: reward = -171.00, steps = 172
23:00:54 [INFO] train episode 745: reward = -133.00, steps = 134
23:00:54 [INFO] train episode 746: reward = -186.00, steps = 187
23:00:55 [INFO] train episode 747: reward = -139.00, steps = 140
23:00:55 [INFO] train episode 748: reward = -174.00, steps = 175
23:00:55 [INFO] train episode 749: reward = -206.00, steps = 207
23:00:55 [INFO] train episode 750: reward = -158.00, steps = 159
23:00:55 [INFO] train episode 751: reward = -235.00, steps = 236
23:00:55 [INFO] train episode 752: reward = -160.00, steps = 161
23:00:56 [INFO] train episode 753: reward = -183.00, steps = 184
23:00:56 [INFO] train episode 754: reward = -181.00, steps = 182
23:00:56 [INFO] train episode 755: reward = -167.00, steps = 168
23:00:56 [INFO] train episode 756: reward = -181.00, steps = 182
23:00:57 [INFO] train episode 757: reward = -500.00, steps = 500
23:00:57 [INFO] train episode 758: reward = -178.00, steps = 179
23:00:57 [INFO] train episode 759: reward = -154.00, steps = 155
23:00:57 [INFO] train episode 760: reward = -176.00, steps = 177
23:00:58 [INFO] train episode 761: reward = -221.00, steps = 222
23:00:58 [INFO] train episode 762: reward = -145.00, steps = 146
23:00:58 [INFO] train episode 763: reward = -175.00, steps = 176
23:00:58 [INFO] train episode 764: reward = -135.00, steps = 136
23:00:58 [INFO] train episode 765: reward = -131.00, steps = 132
23:00:59 [INFO] train episode 766: reward = -177.00, steps = 178
23:00:59 [INFO] train episode 767: reward = -183.00, steps = 184
23:00:59 [INFO] train episode 768: reward = -190.00, steps = 191
23:00:59 [INFO] train episode 769: reward = -154.00, steps = 155
23:00:59 [INFO] train episode 770: reward = -126.00, steps = 127
23:01:00 [INFO] train episode 771: reward = -174.00, steps = 175
23:01:00 [INFO] train episode 772: reward = -203.00, steps = 204
23:01:00 [INFO] train episode 773: reward = -164.00, steps = 165
23:01:00 [INFO] train episode 774: reward = -155.00, steps = 156
23:01:00 [INFO] train episode 775: reward = -123.00, steps = 124
23:01:01 [INFO] train episode 776: reward = -165.00, steps = 166
23:01:01 [INFO] train episode 777: reward = -128.00, steps = 129
23:01:01 [INFO] train episode 778: reward = -178.00, steps = 179
23:01:01 [INFO] train episode 779: reward = -175.00, steps = 176
23:01:01 [INFO] train episode 780: reward = -200.00, steps = 201
23:01:01 [INFO] train episode 781: reward = -155.00, steps = 156
23:01:02 [INFO] train episode 782: reward = -148.00, steps = 149
23:01:02 [INFO] train episode 783: reward = -204.00, steps = 205
23:01:02 [INFO] train episode 784: reward = -149.00, steps = 150
23:01:02 [INFO] train episode 785: reward = -212.00, steps = 213
23:01:02 [INFO] train episode 786: reward = -149.00, steps = 150
23:01:03 [INFO] train episode 787: reward = -150.00, steps = 151
23:01:03 [INFO] train episode 788: reward = -170.00, steps = 171
23:01:03 [INFO] train episode 789: reward = -202.00, steps = 203
23:01:03 [INFO] train episode 790: reward = -233.00, steps = 234
23:01:03 [INFO] train episode 791: reward = -162.00, steps = 163
23:01:04 [INFO] train episode 792: reward = -193.00, steps = 194
23:01:04 [INFO] train episode 793: reward = -187.00, steps = 188
23:01:04 [INFO] train episode 794: reward = -167.00, steps = 168
23:01:04 [INFO] train episode 795: reward = -159.00, steps = 160
23:01:04 [INFO] train episode 796: reward = -166.00, steps = 167
23:01:04 [INFO] train episode 797: reward = -152.00, steps = 153
23:01:05 [INFO] train episode 798: reward = -235.00, steps = 236
23:01:05 [INFO] train episode 799: reward = -175.00, steps = 176
23:01:05 [INFO] train episode 800: reward = -184.00, steps = 185
23:01:05 [INFO] train episode 801: reward = -191.00, steps = 192
23:01:05 [INFO] train episode 802: reward = -140.00, steps = 141
23:01:06 [INFO] train episode 803: reward = -115.00, steps = 116
23:01:06 [INFO] train episode 804: reward = -170.00, steps = 171
23:01:06 [INFO] train episode 805: reward = -161.00, steps = 162
23:01:06 [INFO] train episode 806: reward = -183.00, steps = 184
23:01:06 [INFO] train episode 807: reward = -158.00, steps = 159
23:01:07 [INFO] train episode 808: reward = -193.00, steps = 194
23:01:07 [INFO] train episode 809: reward = -147.00, steps = 148
23:01:07 [INFO] train episode 810: reward = -197.00, steps = 198
23:01:07 [INFO] train episode 811: reward = -166.00, steps = 167
23:01:07 [INFO] train episode 812: reward = -144.00, steps = 145
23:01:07 [INFO] train episode 813: reward = -141.00, steps = 142
23:01:08 [INFO] train episode 814: reward = -121.00, steps = 122
23:01:08 [INFO] train episode 815: reward = -165.00, steps = 166
23:01:08 [INFO] train episode 816: reward = -192.00, steps = 193
23:01:08 [INFO] train episode 817: reward = -169.00, steps = 170
23:01:08 [INFO] train episode 818: reward = -222.00, steps = 223
23:01:09 [INFO] train episode 819: reward = -181.00, steps = 182
23:01:09 [INFO] train episode 820: reward = -161.00, steps = 162
23:01:09 [INFO] train episode 821: reward = -133.00, steps = 134
23:01:09 [INFO] train episode 822: reward = -174.00, steps = 175
23:01:09 [INFO] train episode 823: reward = -154.00, steps = 155
23:01:09 [INFO] train episode 824: reward = -165.00, steps = 166
23:01:10 [INFO] train episode 825: reward = -214.00, steps = 215
23:01:10 [INFO] train episode 826: reward = -177.00, steps = 178
23:01:10 [INFO] train episode 827: reward = -201.00, steps = 202
23:01:10 [INFO] train episode 828: reward = -276.00, steps = 277
23:01:10 [INFO] train episode 829: reward = -212.00, steps = 213
23:01:11 [INFO] train episode 830: reward = -132.00, steps = 133
23:01:11 [INFO] train episode 831: reward = -137.00, steps = 138
23:01:11 [INFO] train episode 832: reward = -164.00, steps = 165
23:01:11 [INFO] train episode 833: reward = -122.00, steps = 123
23:01:11 [INFO] train episode 834: reward = -176.00, steps = 177
23:01:11 [INFO] train episode 835: reward = -161.00, steps = 162
23:01:12 [INFO] train episode 836: reward = -132.00, steps = 133
23:01:12 [INFO] train episode 837: reward = -194.00, steps = 195
23:01:12 [INFO] train episode 838: reward = -162.00, steps = 163
23:01:12 [INFO] train episode 839: reward = -127.00, steps = 128
23:01:12 [INFO] train episode 840: reward = -249.00, steps = 250
23:01:13 [INFO] train episode 841: reward = -132.00, steps = 133
23:01:13 [INFO] train episode 842: reward = -195.00, steps = 196
23:01:13 [INFO] train episode 843: reward = -167.00, steps = 168
23:01:13 [INFO] train episode 844: reward = -189.00, steps = 190
23:01:13 [INFO] train episode 845: reward = -165.00, steps = 166
23:01:14 [INFO] train episode 846: reward = -228.00, steps = 229
23:01:14 [INFO] train episode 847: reward = -199.00, steps = 200
23:01:14 [INFO] train episode 848: reward = -162.00, steps = 163
23:01:14 [INFO] train episode 849: reward = -153.00, steps = 154
23:01:14 [INFO] train episode 850: reward = -164.00, steps = 165
23:01:14 [INFO] train episode 851: reward = -232.00, steps = 233
23:01:15 [INFO] train episode 852: reward = -149.00, steps = 150
23:01:15 [INFO] train episode 853: reward = -251.00, steps = 252
23:01:15 [INFO] train episode 854: reward = -141.00, steps = 142
23:01:15 [INFO] train episode 855: reward = -158.00, steps = 159
23:01:15 [INFO] train episode 856: reward = -165.00, steps = 166
23:01:15 [INFO] train episode 857: reward = -162.00, steps = 163
23:01:16 [INFO] train episode 858: reward = -117.00, steps = 118
23:01:16 [INFO] train episode 859: reward = -173.00, steps = 174
23:01:16 [INFO] train episode 860: reward = -177.00, steps = 178
23:01:16 [INFO] train episode 861: reward = -133.00, steps = 134
23:01:16 [INFO] train episode 862: reward = -129.00, steps = 130
23:01:16 [INFO] train episode 863: reward = -194.00, steps = 195
23:01:17 [INFO] train episode 864: reward = -143.00, steps = 144
23:01:17 [INFO] train episode 865: reward = -196.00, steps = 197
23:01:17 [INFO] train episode 866: reward = -155.00, steps = 156
23:01:17 [INFO] train episode 867: reward = -176.00, steps = 177
23:01:17 [INFO] train episode 868: reward = -147.00, steps = 148
23:01:17 [INFO] train episode 869: reward = -164.00, steps = 165
23:01:18 [INFO] train episode 870: reward = -359.00, steps = 360
23:01:18 [INFO] train episode 871: reward = -145.00, steps = 146
23:01:18 [INFO] train episode 872: reward = -165.00, steps = 166
23:01:18 [INFO] train episode 873: reward = -148.00, steps = 149
23:01:19 [INFO] train episode 874: reward = -212.00, steps = 213
23:01:19 [INFO] train episode 875: reward = -199.00, steps = 200
23:01:19 [INFO] train episode 876: reward = -219.00, steps = 220
23:01:19 [INFO] train episode 877: reward = -171.00, steps = 172
23:01:19 [INFO] train episode 878: reward = -147.00, steps = 148
23:01:19 [INFO] train episode 879: reward = -232.00, steps = 233
23:01:20 [INFO] train episode 880: reward = -148.00, steps = 149
23:01:20 [INFO] train episode 881: reward = -139.00, steps = 140
23:01:20 [INFO] train episode 882: reward = -226.00, steps = 227
23:01:20 [INFO] train episode 883: reward = -151.00, steps = 152
23:01:20 [INFO] train episode 884: reward = -205.00, steps = 206
23:01:20 [INFO] train episode 885: reward = -124.00, steps = 125
23:01:21 [INFO] train episode 886: reward = -197.00, steps = 198
23:01:21 [INFO] train episode 887: reward = -211.00, steps = 212
23:01:21 [INFO] train episode 888: reward = -165.00, steps = 166
23:01:21 [INFO] train episode 889: reward = -172.00, steps = 173
23:01:21 [INFO] train episode 890: reward = -174.00, steps = 175
23:01:22 [INFO] train episode 891: reward = -179.00, steps = 180
23:01:22 [INFO] train episode 892: reward = -160.00, steps = 161
23:01:22 [INFO] train episode 893: reward = -192.00, steps = 193
23:01:22 [INFO] train episode 894: reward = -142.00, steps = 143
23:01:22 [INFO] train episode 895: reward = -208.00, steps = 209
23:01:23 [INFO] train episode 896: reward = -500.00, steps = 500
23:01:23 [INFO] train episode 897: reward = -163.00, steps = 164
23:01:23 [INFO] train episode 898: reward = -155.00, steps = 156
23:01:23 [INFO] train episode 899: reward = -103.00, steps = 104
23:01:23 [INFO] train episode 900: reward = -245.00, steps = 246
23:01:24 [INFO] train episode 901: reward = -152.00, steps = 153
23:01:24 [INFO] train episode 902: reward = -168.00, steps = 169
23:01:24 [INFO] train episode 903: reward = -142.00, steps = 143
23:01:24 [INFO] train episode 904: reward = -213.00, steps = 214
23:01:24 [INFO] train episode 905: reward = -154.00, steps = 155
23:01:24 [INFO] train episode 906: reward = -161.00, steps = 162
23:01:25 [INFO] train episode 907: reward = -136.00, steps = 137
23:01:25 [INFO] train episode 908: reward = -168.00, steps = 169
23:01:25 [INFO] train episode 909: reward = -164.00, steps = 165
23:01:25 [INFO] train episode 910: reward = -148.00, steps = 149
23:01:25 [INFO] train episode 911: reward = -168.00, steps = 169
23:01:25 [INFO] train episode 912: reward = -142.00, steps = 143
23:01:26 [INFO] train episode 913: reward = -155.00, steps = 156
23:01:26 [INFO] train episode 914: reward = -172.00, steps = 173
23:01:26 [INFO] train episode 915: reward = -184.00, steps = 185
23:01:26 [INFO] train episode 916: reward = -204.00, steps = 205
23:01:26 [INFO] train episode 917: reward = -155.00, steps = 156
23:01:27 [INFO] train episode 918: reward = -150.00, steps = 151
23:01:27 [INFO] train episode 919: reward = -175.00, steps = 176
23:01:27 [INFO] train episode 920: reward = -174.00, steps = 175
23:01:27 [INFO] train episode 921: reward = -155.00, steps = 156
23:01:27 [INFO] train episode 922: reward = -152.00, steps = 153
23:01:27 [INFO] train episode 923: reward = -136.00, steps = 137
23:01:27 [INFO] train episode 924: reward = -138.00, steps = 139
23:01:28 [INFO] train episode 925: reward = -150.00, steps = 151
23:01:28 [INFO] train episode 926: reward = -146.00, steps = 147
23:01:28 [INFO] train episode 927: reward = -171.00, steps = 172
23:01:28 [INFO] train episode 928: reward = -208.00, steps = 209
23:01:28 [INFO] train episode 929: reward = -216.00, steps = 217
23:01:29 [INFO] train episode 930: reward = -159.00, steps = 160
23:01:29 [INFO] train episode 931: reward = -180.00, steps = 181
23:01:29 [INFO] train episode 932: reward = -189.00, steps = 190
23:01:29 [INFO] train episode 933: reward = -143.00, steps = 144
23:01:29 [INFO] train episode 934: reward = -142.00, steps = 143
23:01:29 [INFO] train episode 935: reward = -164.00, steps = 165
23:01:30 [INFO] train episode 936: reward = -145.00, steps = 146
23:01:30 [INFO] train episode 937: reward = -154.00, steps = 155
23:01:30 [INFO] train episode 938: reward = -146.00, steps = 147
23:01:30 [INFO] train episode 939: reward = -178.00, steps = 179
23:01:30 [INFO] train episode 940: reward = -183.00, steps = 184
23:01:30 [INFO] train episode 941: reward = -226.00, steps = 227
23:01:31 [INFO] train episode 942: reward = -182.00, steps = 183
23:01:31 [INFO] train episode 943: reward = -167.00, steps = 168
23:01:31 [INFO] train episode 944: reward = -228.00, steps = 229
23:01:31 [INFO] train episode 945: reward = -139.00, steps = 140
23:01:31 [INFO] train episode 946: reward = -155.00, steps = 156
23:01:31 [INFO] train episode 947: reward = -184.00, steps = 185
23:01:32 [INFO] train episode 948: reward = -173.00, steps = 174
23:01:32 [INFO] train episode 949: reward = -191.00, steps = 192
23:01:32 [INFO] train episode 950: reward = -170.00, steps = 171
23:01:32 [INFO] train episode 951: reward = -194.00, steps = 195
23:01:32 [INFO] train episode 952: reward = -160.00, steps = 161
23:01:33 [INFO] train episode 953: reward = -150.00, steps = 151
23:01:33 [INFO] train episode 954: reward = -168.00, steps = 169
23:01:33 [INFO] train episode 955: reward = -190.00, steps = 191
23:01:33 [INFO] train episode 956: reward = -152.00, steps = 153
23:01:33 [INFO] train episode 957: reward = -142.00, steps = 143
23:01:33 [INFO] train episode 958: reward = -142.00, steps = 143
23:01:34 [INFO] train episode 959: reward = -230.00, steps = 231
23:01:34 [INFO] train episode 960: reward = -188.00, steps = 189
23:01:34 [INFO] train episode 961: reward = -179.00, steps = 180
23:01:34 [INFO] train episode 962: reward = -319.00, steps = 320
23:01:35 [INFO] train episode 963: reward = -170.00, steps = 171
23:01:35 [INFO] train episode 964: reward = -165.00, steps = 166
23:01:35 [INFO] train episode 965: reward = -128.00, steps = 129
23:01:35 [INFO] train episode 966: reward = -144.00, steps = 145
23:01:35 [INFO] train episode 967: reward = -202.00, steps = 203
23:01:35 [INFO] train episode 968: reward = -154.00, steps = 155
23:01:36 [INFO] train episode 969: reward = -169.00, steps = 170
23:01:36 [INFO] train episode 970: reward = -179.00, steps = 180
23:01:36 [INFO] train episode 971: reward = -158.00, steps = 159
23:01:36 [INFO] train episode 972: reward = -159.00, steps = 160
23:01:36 [INFO] train episode 973: reward = -149.00, steps = 150
23:01:36 [INFO] train episode 974: reward = -171.00, steps = 172
23:01:37 [INFO] train episode 975: reward = -206.00, steps = 207
23:01:37 [INFO] train episode 976: reward = -141.00, steps = 142
23:01:37 [INFO] train episode 977: reward = -215.00, steps = 216
23:01:37 [INFO] train episode 978: reward = -169.00, steps = 170
23:01:37 [INFO] train episode 979: reward = -134.00, steps = 135
23:01:38 [INFO] train episode 980: reward = -172.00, steps = 173
23:01:38 [INFO] train episode 981: reward = -202.00, steps = 203
23:01:38 [INFO] train episode 982: reward = -140.00, steps = 141
23:01:38 [INFO] train episode 983: reward = -144.00, steps = 145
23:01:38 [INFO] train episode 984: reward = -214.00, steps = 215
23:01:39 [INFO] train episode 985: reward = -170.00, steps = 171
23:01:39 [INFO] train episode 986: reward = -238.00, steps = 239
23:01:39 [INFO] train episode 987: reward = -166.00, steps = 167
23:01:39 [INFO] train episode 988: reward = -171.00, steps = 172
23:01:39 [INFO] train episode 989: reward = -117.00, steps = 118
23:01:39 [INFO] train episode 990: reward = -170.00, steps = 171
23:01:40 [INFO] train episode 991: reward = -159.00, steps = 160
23:01:40 [INFO] train episode 992: reward = -185.00, steps = 186
23:01:40 [INFO] train episode 993: reward = -152.00, steps = 153
23:01:40 [INFO] train episode 994: reward = -166.00, steps = 167
23:01:40 [INFO] train episode 995: reward = -189.00, steps = 190
23:01:41 [INFO] train episode 996: reward = -212.00, steps = 213
23:01:41 [INFO] train episode 997: reward = -181.00, steps = 182
23:01:41 [INFO] train episode 998: reward = -151.00, steps = 152
23:01:41 [INFO] train episode 999: reward = -150.00, steps = 151
23:01:41 [INFO] train episode 1000: reward = -158.00, steps = 159
23:01:41 [INFO] train episode 1001: reward = -212.00, steps = 213
23:01:42 [INFO] train episode 1002: reward = -155.00, steps = 156
23:01:42 [INFO] train episode 1003: reward = -240.00, steps = 241
23:01:42 [INFO] train episode 1004: reward = -143.00, steps = 144
23:01:42 [INFO] train episode 1005: reward = -262.00, steps = 263
23:01:42 [INFO] train episode 1006: reward = -181.00, steps = 182
23:01:43 [INFO] train episode 1007: reward = -172.00, steps = 173
23:01:43 [INFO] train episode 1008: reward = -319.00, steps = 320
23:01:43 [INFO] train episode 1009: reward = -143.00, steps = 144
23:01:43 [INFO] train episode 1010: reward = -167.00, steps = 168
23:01:43 [INFO] train episode 1011: reward = -151.00, steps = 152
23:01:44 [INFO] train episode 1012: reward = -220.00, steps = 221
23:01:44 [INFO] train episode 1013: reward = -402.00, steps = 403
23:01:44 [INFO] train episode 1014: reward = -278.00, steps = 279
23:01:44 [INFO] train episode 1015: reward = -145.00, steps = 146
23:01:45 [INFO] train episode 1016: reward = -166.00, steps = 167
23:01:45 [INFO] train episode 1017: reward = -232.00, steps = 233
23:01:45 [INFO] train episode 1018: reward = -158.00, steps = 159
23:01:45 [INFO] train episode 1019: reward = -157.00, steps = 158
23:01:45 [INFO] train episode 1020: reward = -169.00, steps = 170
23:01:46 [INFO] train episode 1021: reward = -158.00, steps = 159
23:01:46 [INFO] train episode 1022: reward = -196.00, steps = 197
23:01:46 [INFO] train episode 1023: reward = -197.00, steps = 198
23:01:46 [INFO] train episode 1024: reward = -172.00, steps = 173
23:01:46 [INFO] train episode 1025: reward = -192.00, steps = 193
23:01:47 [INFO] train episode 1026: reward = -188.00, steps = 189
23:01:47 [INFO] train episode 1027: reward = -130.00, steps = 131
23:01:47 [INFO] train episode 1028: reward = -212.00, steps = 213
23:01:47 [INFO] train episode 1029: reward = -188.00, steps = 189
23:01:47 [INFO] train episode 1030: reward = -134.00, steps = 135
23:01:47 [INFO] train episode 1031: reward = -214.00, steps = 215
23:01:48 [INFO] train episode 1032: reward = -164.00, steps = 165
23:01:48 [INFO] train episode 1033: reward = -144.00, steps = 145
23:01:48 [INFO] train episode 1034: reward = -250.00, steps = 251
23:01:48 [INFO] train episode 1035: reward = -229.00, steps = 230
23:01:48 [INFO] train episode 1036: reward = -163.00, steps = 164
23:01:49 [INFO] train episode 1037: reward = -144.00, steps = 145
23:01:49 [INFO] train episode 1038: reward = -150.00, steps = 151
23:01:49 [INFO] train episode 1039: reward = -195.00, steps = 196
23:01:49 [INFO] train episode 1040: reward = -182.00, steps = 183
23:01:49 [INFO] train episode 1041: reward = -135.00, steps = 136
23:01:50 [INFO] train episode 1042: reward = -190.00, steps = 191
23:01:50 [INFO] train episode 1043: reward = -243.00, steps = 244
23:01:50 [INFO] train episode 1044: reward = -199.00, steps = 200
23:01:50 [INFO] train episode 1045: reward = -187.00, steps = 188
23:01:50 [INFO] train episode 1046: reward = -203.00, steps = 204
23:01:51 [INFO] train episode 1047: reward = -164.00, steps = 165
23:01:51 [INFO] train episode 1048: reward = -162.00, steps = 163
23:01:51 [INFO] train episode 1049: reward = -176.00, steps = 177
23:01:51 [INFO] train episode 1050: reward = -167.00, steps = 168
23:01:51 [INFO] train episode 1051: reward = -176.00, steps = 177
23:01:51 [INFO] train episode 1052: reward = -197.00, steps = 198
23:01:52 [INFO] train episode 1053: reward = -181.00, steps = 182
23:01:52 [INFO] train episode 1054: reward = -149.00, steps = 150
23:01:52 [INFO] train episode 1055: reward = -140.00, steps = 141
23:01:52 [INFO] train episode 1056: reward = -151.00, steps = 152
23:01:52 [INFO] train episode 1057: reward = -179.00, steps = 180
23:01:52 [INFO] train episode 1058: reward = -170.00, steps = 171
23:01:53 [INFO] train episode 1059: reward = -155.00, steps = 156
23:01:53 [INFO] train episode 1060: reward = -156.00, steps = 157
23:01:53 [INFO] train episode 1061: reward = -171.00, steps = 172
23:01:53 [INFO] train episode 1062: reward = -158.00, steps = 159
23:01:53 [INFO] train episode 1063: reward = -172.00, steps = 173
23:01:54 [INFO] train episode 1064: reward = -218.00, steps = 219
23:01:54 [INFO] train episode 1065: reward = -199.00, steps = 200
23:01:54 [INFO] train episode 1066: reward = -149.00, steps = 150
23:01:54 [INFO] train episode 1067: reward = -140.00, steps = 141
23:01:54 [INFO] train episode 1068: reward = -146.00, steps = 147
23:01:54 [INFO] train episode 1069: reward = -177.00, steps = 178
23:01:55 [INFO] train episode 1070: reward = -198.00, steps = 199
23:01:55 [INFO] train episode 1071: reward = -167.00, steps = 168
23:01:55 [INFO] train episode 1072: reward = -149.00, steps = 150
23:01:55 [INFO] train episode 1073: reward = -170.00, steps = 171
23:01:55 [INFO] train episode 1074: reward = -139.00, steps = 140
23:01:56 [INFO] train episode 1075: reward = -182.00, steps = 183
23:01:56 [INFO] train episode 1076: reward = -193.00, steps = 194
23:01:56 [INFO] train episode 1077: reward = -215.00, steps = 216
23:01:56 [INFO] train episode 1078: reward = -203.00, steps = 204
23:01:56 [INFO] train episode 1079: reward = -182.00, steps = 183
23:01:57 [INFO] train episode 1080: reward = -209.00, steps = 210
23:01:57 [INFO] train episode 1081: reward = -183.00, steps = 184
23:01:57 [INFO] train episode 1082: reward = -177.00, steps = 178
23:01:57 [INFO] train episode 1083: reward = -163.00, steps = 164
23:01:57 [INFO] train episode 1084: reward = -183.00, steps = 184
23:01:57 [INFO] train episode 1085: reward = -183.00, steps = 184
23:01:58 [INFO] train episode 1086: reward = -207.00, steps = 208
23:01:58 [INFO] train episode 1087: reward = -181.00, steps = 182
23:01:58 [INFO] train episode 1088: reward = -186.00, steps = 187
23:01:58 [INFO] train episode 1089: reward = -219.00, steps = 220
23:01:58 [INFO] train episode 1090: reward = -148.00, steps = 149
23:01:59 [INFO] train episode 1091: reward = -156.00, steps = 157
23:01:59 [INFO] train episode 1092: reward = -174.00, steps = 175
23:01:59 [INFO] train episode 1093: reward = -191.00, steps = 192
23:01:59 [INFO] train episode 1094: reward = -208.00, steps = 209
23:01:59 [INFO] train episode 1095: reward = -317.00, steps = 318
23:02:00 [INFO] train episode 1096: reward = -224.00, steps = 225
23:02:00 [INFO] train episode 1097: reward = -151.00, steps = 152
23:02:00 [INFO] train episode 1098: reward = -208.00, steps = 209
23:02:00 [INFO] train episode 1099: reward = -163.00, steps = 164
23:02:00 [INFO] train episode 1100: reward = -161.00, steps = 162
23:02:01 [INFO] train episode 1101: reward = -164.00, steps = 165
23:02:01 [INFO] train episode 1102: reward = -188.00, steps = 189
23:02:01 [INFO] train episode 1103: reward = -235.00, steps = 236
23:02:01 [INFO] train episode 1104: reward = -211.00, steps = 212
23:02:01 [INFO] train episode 1105: reward = -267.00, steps = 268
23:02:02 [INFO] train episode 1106: reward = -166.00, steps = 167
23:02:02 [INFO] train episode 1107: reward = -163.00, steps = 164
23:02:02 [INFO] train episode 1108: reward = -150.00, steps = 151
23:02:02 [INFO] train episode 1109: reward = -228.00, steps = 229
23:02:02 [INFO] train episode 1110: reward = -222.00, steps = 223
23:02:03 [INFO] train episode 1111: reward = -186.00, steps = 187
23:02:03 [INFO] train episode 1112: reward = -237.00, steps = 238
23:02:03 [INFO] train episode 1113: reward = -207.00, steps = 208
23:02:03 [INFO] train episode 1114: reward = -183.00, steps = 184
23:02:03 [INFO] train episode 1115: reward = -152.00, steps = 153
23:02:04 [INFO] train episode 1116: reward = -128.00, steps = 129
23:02:04 [INFO] train episode 1117: reward = -277.00, steps = 278
23:02:04 [INFO] train episode 1118: reward = -211.00, steps = 212
23:02:04 [INFO] train episode 1119: reward = -144.00, steps = 145
23:02:04 [INFO] train episode 1120: reward = -156.00, steps = 157
23:02:04 [INFO] train episode 1121: reward = -186.00, steps = 187
23:02:05 [INFO] train episode 1122: reward = -162.00, steps = 163
23:02:05 [INFO] train episode 1123: reward = -141.00, steps = 142
23:02:05 [INFO] train episode 1124: reward = -185.00, steps = 186
23:02:05 [INFO] train episode 1125: reward = -177.00, steps = 178
23:02:05 [INFO] train episode 1126: reward = -186.00, steps = 187
23:02:06 [INFO] train episode 1127: reward = -168.00, steps = 169
23:02:06 [INFO] train episode 1128: reward = -164.00, steps = 165
23:02:06 [INFO] train episode 1129: reward = -172.00, steps = 173
23:02:06 [INFO] train episode 1130: reward = -155.00, steps = 156
23:02:06 [INFO] train episode 1131: reward = -176.00, steps = 177
23:02:06 [INFO] train episode 1132: reward = -152.00, steps = 153
23:02:07 [INFO] train episode 1133: reward = -175.00, steps = 176
23:02:07 [INFO] train episode 1134: reward = -173.00, steps = 174
23:02:07 [INFO] train episode 1135: reward = -217.00, steps = 218
23:02:07 [INFO] train episode 1136: reward = -146.00, steps = 147
23:02:07 [INFO] train episode 1137: reward = -170.00, steps = 171
23:02:08 [INFO] train episode 1138: reward = -179.00, steps = 180
23:02:08 [INFO] train episode 1139: reward = -140.00, steps = 141
23:02:08 [INFO] train episode 1140: reward = -151.00, steps = 152
23:02:08 [INFO] train episode 1141: reward = -179.00, steps = 180
23:02:08 [INFO] train episode 1142: reward = -142.00, steps = 143
23:02:08 [INFO] train episode 1143: reward = -192.00, steps = 193
23:02:09 [INFO] train episode 1144: reward = -247.00, steps = 248
23:02:09 [INFO] train episode 1145: reward = -147.00, steps = 148
23:02:09 [INFO] train episode 1146: reward = -125.00, steps = 126
23:02:09 [INFO] train episode 1147: reward = -141.00, steps = 142
23:02:09 [INFO] train episode 1148: reward = -129.00, steps = 130
23:02:09 [INFO] train episode 1149: reward = -192.00, steps = 193
23:02:10 [INFO] train episode 1150: reward = -251.00, steps = 252
23:02:10 [INFO] train episode 1151: reward = -183.00, steps = 184
23:02:10 [INFO] train episode 1152: reward = -257.00, steps = 258
23:02:10 [INFO] train episode 1153: reward = -202.00, steps = 203
23:02:10 [INFO] train episode 1154: reward = -142.00, steps = 143
23:02:11 [INFO] train episode 1155: reward = -127.00, steps = 128
23:02:11 [INFO] train episode 1156: reward = -150.00, steps = 151
23:02:11 [INFO] train episode 1157: reward = -170.00, steps = 171
23:02:11 [INFO] train episode 1158: reward = -179.00, steps = 180
23:02:11 [INFO] train episode 1159: reward = -146.00, steps = 147
23:02:11 [INFO] train episode 1160: reward = -117.00, steps = 118
23:02:12 [INFO] train episode 1161: reward = -189.00, steps = 190
23:02:12 [INFO] train episode 1162: reward = -358.00, steps = 359
23:02:12 [INFO] train episode 1163: reward = -131.00, steps = 132
23:02:12 [INFO] train episode 1164: reward = -171.00, steps = 172
23:02:12 [INFO] train episode 1165: reward = -134.00, steps = 135
23:02:13 [INFO] train episode 1166: reward = -189.00, steps = 190
23:02:13 [INFO] train episode 1167: reward = -148.00, steps = 149
23:02:13 [INFO] train episode 1168: reward = -178.00, steps = 179
23:02:13 [INFO] train episode 1169: reward = -240.00, steps = 241
23:02:13 [INFO] train episode 1170: reward = -118.00, steps = 119
23:02:14 [INFO] train episode 1171: reward = -326.00, steps = 327
23:02:14 [INFO] train episode 1172: reward = -153.00, steps = 154
23:02:14 [INFO] train episode 1173: reward = -173.00, steps = 174
23:02:14 [INFO] train episode 1174: reward = -137.00, steps = 138
23:02:15 [INFO] train episode 1175: reward = -393.00, steps = 394
23:02:15 [INFO] train episode 1176: reward = -168.00, steps = 169
23:02:15 [INFO] train episode 1177: reward = -258.00, steps = 259
23:02:15 [INFO] train episode 1178: reward = -195.00, steps = 196
23:02:16 [INFO] train episode 1179: reward = -152.00, steps = 153
23:02:16 [INFO] train episode 1180: reward = -232.00, steps = 233
23:02:16 [INFO] train episode 1181: reward = -156.00, steps = 157
23:02:16 [INFO] train episode 1182: reward = -161.00, steps = 162
23:02:16 [INFO] train episode 1183: reward = -137.00, steps = 138
23:02:16 [INFO] train episode 1184: reward = -168.00, steps = 169
23:02:17 [INFO] train episode 1185: reward = -147.00, steps = 148
23:02:17 [INFO] train episode 1186: reward = -232.00, steps = 233
23:02:17 [INFO] train episode 1187: reward = -173.00, steps = 174
23:02:17 [INFO] train episode 1188: reward = -150.00, steps = 151
23:02:17 [INFO] train episode 1189: reward = -194.00, steps = 195
23:02:17 [INFO] train episode 1190: reward = -147.00, steps = 148
23:02:18 [INFO] train episode 1191: reward = -200.00, steps = 201
23:02:18 [INFO] train episode 1192: reward = -202.00, steps = 203
23:02:18 [INFO] train episode 1193: reward = -304.00, steps = 305
23:02:18 [INFO] train episode 1194: reward = -116.00, steps = 117
23:02:19 [INFO] train episode 1195: reward = -198.00, steps = 199
23:02:19 [INFO] train episode 1196: reward = -152.00, steps = 153
23:02:19 [INFO] train episode 1197: reward = -211.00, steps = 212
23:02:19 [INFO] train episode 1198: reward = -118.00, steps = 119
23:02:19 [INFO] train episode 1199: reward = -174.00, steps = 175
23:02:19 [INFO] train episode 1200: reward = -157.00, steps = 158
23:02:20 [INFO] train episode 1201: reward = -123.00, steps = 124
23:02:20 [INFO] train episode 1202: reward = -145.00, steps = 146
23:02:20 [INFO] train episode 1203: reward = -225.00, steps = 226
23:02:20 [INFO] train episode 1204: reward = -152.00, steps = 153
23:02:20 [INFO] train episode 1205: reward = -167.00, steps = 168
23:02:20 [INFO] train episode 1206: reward = -149.00, steps = 150
23:02:21 [INFO] train episode 1207: reward = -151.00, steps = 152
23:02:21 [INFO] train episode 1208: reward = -150.00, steps = 151
23:02:21 [INFO] train episode 1209: reward = -196.00, steps = 197
23:02:21 [INFO] train episode 1210: reward = -248.00, steps = 249
23:02:21 [INFO] train episode 1211: reward = -147.00, steps = 148
23:02:21 [INFO] train episode 1212: reward = -277.00, steps = 278
23:02:22 [INFO] train episode 1213: reward = -204.00, steps = 205
23:02:22 [INFO] train episode 1214: reward = -143.00, steps = 144
23:02:22 [INFO] train episode 1215: reward = -159.00, steps = 160
23:02:22 [INFO] train episode 1216: reward = -216.00, steps = 217
23:02:22 [INFO] train episode 1217: reward = -165.00, steps = 166
23:02:23 [INFO] train episode 1218: reward = -147.00, steps = 148
23:02:23 [INFO] train episode 1219: reward = -149.00, steps = 150
23:02:23 [INFO] train episode 1220: reward = -126.00, steps = 127
23:02:23 [INFO] train episode 1221: reward = -164.00, steps = 165
23:02:23 [INFO] train episode 1222: reward = -127.00, steps = 128
23:02:23 [INFO] train episode 1223: reward = -146.00, steps = 147
23:02:23 [INFO] train episode 1224: reward = -169.00, steps = 170
23:02:24 [INFO] train episode 1225: reward = -137.00, steps = 138
23:02:24 [INFO] train episode 1226: reward = -145.00, steps = 146
23:02:24 [INFO] train episode 1227: reward = -213.00, steps = 214
23:02:24 [INFO] train episode 1228: reward = -215.00, steps = 216
23:02:24 [INFO] train episode 1229: reward = -158.00, steps = 159
23:02:25 [INFO] train episode 1230: reward = -158.00, steps = 159
23:02:25 [INFO] train episode 1231: reward = -176.00, steps = 177
23:02:25 [INFO] train episode 1232: reward = -146.00, steps = 147
23:02:25 [INFO] train episode 1233: reward = -153.00, steps = 154
23:02:25 [INFO] train episode 1234: reward = -266.00, steps = 267
23:02:25 [INFO] train episode 1235: reward = -177.00, steps = 178
23:02:26 [INFO] train episode 1236: reward = -152.00, steps = 153
23:02:26 [INFO] train episode 1237: reward = -171.00, steps = 172
23:02:26 [INFO] train episode 1238: reward = -159.00, steps = 160
23:02:26 [INFO] train episode 1239: reward = -153.00, steps = 154
23:02:26 [INFO] train episode 1240: reward = -168.00, steps = 169
23:02:26 [INFO] train episode 1241: reward = -161.00, steps = 162
23:02:27 [INFO] train episode 1242: reward = -161.00, steps = 162
23:02:27 [INFO] train episode 1243: reward = -125.00, steps = 126
23:02:27 [INFO] train episode 1244: reward = -157.00, steps = 158
23:02:27 [INFO] train episode 1245: reward = -169.00, steps = 170
23:02:27 [INFO] train episode 1246: reward = -126.00, steps = 127
23:02:27 [INFO] train episode 1247: reward = -183.00, steps = 184
23:02:28 [INFO] train episode 1248: reward = -171.00, steps = 172
23:02:28 [INFO] train episode 1249: reward = -140.00, steps = 141
23:02:28 [INFO] train episode 1250: reward = -166.00, steps = 167
23:02:28 [INFO] train episode 1251: reward = -192.00, steps = 193
23:02:28 [INFO] train episode 1252: reward = -277.00, steps = 278
23:02:29 [INFO] train episode 1253: reward = -179.00, steps = 180
23:02:29 [INFO] train episode 1254: reward = -123.00, steps = 124
23:02:29 [INFO] train episode 1255: reward = -177.00, steps = 178
23:02:29 [INFO] train episode 1256: reward = -189.00, steps = 190
23:02:29 [INFO] train episode 1257: reward = -171.00, steps = 172
23:02:29 [INFO] train episode 1258: reward = -164.00, steps = 165
23:02:30 [INFO] train episode 1259: reward = -227.00, steps = 228
23:02:30 [INFO] train episode 1260: reward = -134.00, steps = 135
23:02:30 [INFO] train episode 1261: reward = -184.00, steps = 185
23:02:30 [INFO] train episode 1262: reward = -150.00, steps = 151
23:02:30 [INFO] train episode 1263: reward = -167.00, steps = 168
23:02:31 [INFO] train episode 1264: reward = -164.00, steps = 165
23:02:31 [INFO] train episode 1265: reward = -167.00, steps = 168
23:02:31 [INFO] train episode 1266: reward = -195.00, steps = 196
23:02:31 [INFO] train episode 1267: reward = -154.00, steps = 155
23:02:31 [INFO] train episode 1268: reward = -166.00, steps = 167
23:02:31 [INFO] train episode 1269: reward = -147.00, steps = 148
23:02:32 [INFO] train episode 1270: reward = -214.00, steps = 215
23:02:32 [INFO] train episode 1271: reward = -126.00, steps = 127
23:02:32 [INFO] train episode 1272: reward = -153.00, steps = 154
23:02:32 [INFO] train episode 1273: reward = -149.00, steps = 150
23:02:32 [INFO] train episode 1274: reward = -142.00, steps = 143
23:02:32 [INFO] train episode 1275: reward = -208.00, steps = 209
23:02:33 [INFO] train episode 1276: reward = -231.00, steps = 232
23:02:33 [INFO] train episode 1277: reward = -151.00, steps = 152
23:02:33 [INFO] train episode 1278: reward = -155.00, steps = 156
23:02:33 [INFO] train episode 1279: reward = -198.00, steps = 199
23:02:33 [INFO] train episode 1280: reward = -125.00, steps = 126
23:02:33 [INFO] train episode 1281: reward = -155.00, steps = 156
23:02:34 [INFO] train episode 1282: reward = -152.00, steps = 153
23:02:34 [INFO] train episode 1283: reward = -154.00, steps = 155
23:02:34 [INFO] train episode 1284: reward = -137.00, steps = 138
23:02:34 [INFO] train episode 1285: reward = -243.00, steps = 244
23:02:34 [INFO] train episode 1286: reward = -112.00, steps = 113
23:02:34 [INFO] train episode 1287: reward = -162.00, steps = 163
23:02:35 [INFO] train episode 1288: reward = -149.00, steps = 150
23:02:35 [INFO] train episode 1289: reward = -124.00, steps = 125
23:02:35 [INFO] train episode 1290: reward = -172.00, steps = 173
23:02:35 [INFO] train episode 1291: reward = -183.00, steps = 184
23:02:35 [INFO] train episode 1292: reward = -185.00, steps = 186
23:02:35 [INFO] train episode 1293: reward = -147.00, steps = 148
23:02:35 [INFO] train episode 1294: reward = -162.00, steps = 163
23:02:36 [INFO] train episode 1295: reward = -183.00, steps = 184
23:02:36 [INFO] train episode 1296: reward = -177.00, steps = 178
23:02:36 [INFO] train episode 1297: reward = -148.00, steps = 149
23:02:36 [INFO] train episode 1298: reward = -127.00, steps = 128
23:02:36 [INFO] train episode 1299: reward = -191.00, steps = 192
23:02:37 [INFO] train episode 1300: reward = -224.00, steps = 225
23:02:37 [INFO] train episode 1301: reward = -172.00, steps = 173
23:02:37 [INFO] train episode 1302: reward = -154.00, steps = 155
23:02:37 [INFO] train episode 1303: reward = -158.00, steps = 159
23:02:37 [INFO] train episode 1304: reward = -157.00, steps = 158
23:02:37 [INFO] train episode 1305: reward = -153.00, steps = 154
23:02:38 [INFO] train episode 1306: reward = -143.00, steps = 144
23:02:38 [INFO] train episode 1307: reward = -179.00, steps = 180
23:02:38 [INFO] train episode 1308: reward = -143.00, steps = 144
23:02:38 [INFO] train episode 1309: reward = -143.00, steps = 144
23:02:38 [INFO] train episode 1310: reward = -149.00, steps = 150
23:02:38 [INFO] train episode 1311: reward = -142.00, steps = 143
23:02:38 [INFO] train episode 1312: reward = -165.00, steps = 166
23:02:39 [INFO] train episode 1313: reward = -158.00, steps = 159
23:02:39 [INFO] train episode 1314: reward = -194.00, steps = 195
23:02:39 [INFO] train episode 1315: reward = -137.00, steps = 138
23:02:39 [INFO] train episode 1316: reward = -146.00, steps = 147
23:02:39 [INFO] train episode 1317: reward = -161.00, steps = 162
23:02:39 [INFO] train episode 1318: reward = -149.00, steps = 150
23:02:40 [INFO] train episode 1319: reward = -168.00, steps = 169
23:02:40 [INFO] train episode 1320: reward = -268.00, steps = 269
23:02:40 [INFO] train episode 1321: reward = -153.00, steps = 154
23:02:40 [INFO] train episode 1322: reward = -147.00, steps = 148
23:02:40 [INFO] train episode 1323: reward = -160.00, steps = 161
23:02:40 [INFO] train episode 1324: reward = -143.00, steps = 144
23:02:41 [INFO] train episode 1325: reward = -170.00, steps = 171
23:02:41 [INFO] train episode 1326: reward = -161.00, steps = 162
23:02:41 [INFO] train episode 1327: reward = -182.00, steps = 183
23:02:41 [INFO] train episode 1328: reward = -212.00, steps = 213
23:02:41 [INFO] train episode 1329: reward = -164.00, steps = 165
23:02:41 [INFO] train episode 1330: reward = -142.00, steps = 143
23:02:42 [INFO] train episode 1331: reward = -187.00, steps = 188
23:02:42 [INFO] train episode 1332: reward = -139.00, steps = 140
23:02:42 [INFO] train episode 1333: reward = -136.00, steps = 137
23:02:42 [INFO] train episode 1334: reward = -116.00, steps = 117
23:02:42 [INFO] train episode 1335: reward = -153.00, steps = 154
23:02:42 [INFO] train episode 1336: reward = -149.00, steps = 150
23:02:43 [INFO] train episode 1337: reward = -189.00, steps = 190
23:02:43 [INFO] train episode 1338: reward = -185.00, steps = 186
23:02:43 [INFO] train episode 1339: reward = -141.00, steps = 142
23:02:43 [INFO] train episode 1340: reward = -124.00, steps = 125
23:02:43 [INFO] train episode 1341: reward = -268.00, steps = 269
23:02:44 [INFO] train episode 1342: reward = -136.00, steps = 137
23:02:44 [INFO] train episode 1343: reward = -196.00, steps = 197
23:02:44 [INFO] train episode 1344: reward = -217.00, steps = 218
23:02:44 [INFO] train episode 1345: reward = -163.00, steps = 164
23:02:44 [INFO] train episode 1346: reward = -136.00, steps = 137
23:02:45 [INFO] train episode 1347: reward = -146.00, steps = 147
23:02:45 [INFO] train episode 1348: reward = -136.00, steps = 137
23:02:45 [INFO] train episode 1349: reward = -158.00, steps = 159
23:02:45 [INFO] train episode 1350: reward = -216.00, steps = 217
23:02:45 [INFO] train episode 1351: reward = -187.00, steps = 188
23:02:45 [INFO] train episode 1352: reward = -126.00, steps = 127
23:02:45 [INFO] train episode 1353: reward = -142.00, steps = 143
23:02:46 [INFO] train episode 1354: reward = -117.00, steps = 118
23:02:46 [INFO] train episode 1355: reward = -113.00, steps = 114
23:02:46 [INFO] train episode 1356: reward = -162.00, steps = 163
23:02:46 [INFO] train episode 1357: reward = -128.00, steps = 129
23:02:46 [INFO] train episode 1358: reward = -174.00, steps = 175
23:02:46 [INFO] train episode 1359: reward = -138.00, steps = 139
23:02:47 [INFO] train episode 1360: reward = -160.00, steps = 161
23:02:47 [INFO] train episode 1361: reward = -134.00, steps = 135
23:02:47 [INFO] train episode 1362: reward = -207.00, steps = 208
23:02:47 [INFO] train episode 1363: reward = -143.00, steps = 144
23:02:47 [INFO] train episode 1364: reward = -166.00, steps = 167
23:02:47 [INFO] train episode 1365: reward = -221.00, steps = 222
23:02:48 [INFO] train episode 1366: reward = -180.00, steps = 181
23:02:48 [INFO] train episode 1367: reward = -151.00, steps = 152
23:02:48 [INFO] train episode 1368: reward = -168.00, steps = 169
23:02:48 [INFO] train episode 1369: reward = -142.00, steps = 143
23:02:48 [INFO] train episode 1370: reward = -142.00, steps = 143
23:02:48 [INFO] train episode 1371: reward = -168.00, steps = 169
23:02:49 [INFO] train episode 1372: reward = -180.00, steps = 181
23:02:49 [INFO] train episode 1373: reward = -133.00, steps = 134
23:02:49 [INFO] train episode 1374: reward = -207.00, steps = 208
23:02:49 [INFO] train episode 1375: reward = -178.00, steps = 179
23:02:49 [INFO] train episode 1376: reward = -151.00, steps = 152
23:02:49 [INFO] train episode 1377: reward = -115.00, steps = 116
23:02:50 [INFO] train episode 1378: reward = -159.00, steps = 160
23:02:50 [INFO] train episode 1379: reward = -150.00, steps = 151
23:02:50 [INFO] train episode 1380: reward = -110.00, steps = 111
23:02:50 [INFO] train episode 1381: reward = -117.00, steps = 118
23:02:50 [INFO] train episode 1382: reward = -135.00, steps = 136
23:02:50 [INFO] train episode 1383: reward = -180.00, steps = 181
23:02:50 [INFO] train episode 1384: reward = -136.00, steps = 137
23:02:51 [INFO] train episode 1385: reward = -147.00, steps = 148
23:02:51 [INFO] train episode 1386: reward = -108.00, steps = 109
23:02:51 [INFO] train episode 1387: reward = -123.00, steps = 124
23:02:51 [INFO] train episode 1388: reward = -153.00, steps = 154
23:02:51 [INFO] train episode 1389: reward = -153.00, steps = 154
23:02:51 [INFO] train episode 1390: reward = -132.00, steps = 133
23:02:52 [INFO] train episode 1391: reward = -173.00, steps = 174
23:02:52 [INFO] train episode 1392: reward = -141.00, steps = 142
23:02:52 [INFO] train episode 1393: reward = -175.00, steps = 176
23:02:52 [INFO] train episode 1394: reward = -228.00, steps = 229
23:02:52 [INFO] train episode 1395: reward = -158.00, steps = 159
23:02:52 [INFO] train episode 1396: reward = -150.00, steps = 151
23:02:53 [INFO] train episode 1397: reward = -225.00, steps = 226
23:02:53 [INFO] train episode 1398: reward = -180.00, steps = 181
23:02:53 [INFO] train episode 1399: reward = -172.00, steps = 173
23:02:53 [INFO] train episode 1400: reward = -176.00, steps = 177
23:02:53 [INFO] train episode 1401: reward = -152.00, steps = 153
23:02:54 [INFO] train episode 1402: reward = -127.00, steps = 128
23:02:54 [INFO] train episode 1403: reward = -224.00, steps = 225
23:02:54 [INFO] train episode 1404: reward = -131.00, steps = 132
23:02:54 [INFO] train episode 1405: reward = -160.00, steps = 161
23:02:54 [INFO] train episode 1406: reward = -178.00, steps = 179
23:02:54 [INFO] train episode 1407: reward = -152.00, steps = 153
23:02:55 [INFO] train episode 1408: reward = -121.00, steps = 122
23:02:55 [INFO] train episode 1409: reward = -177.00, steps = 178
23:02:55 [INFO] train episode 1410: reward = -155.00, steps = 156
23:02:55 [INFO] train episode 1411: reward = -140.00, steps = 141
23:02:55 [INFO] train episode 1412: reward = -184.00, steps = 185
23:02:56 [INFO] train episode 1413: reward = -247.00, steps = 248
23:02:56 [INFO] train episode 1414: reward = -170.00, steps = 171
23:02:56 [INFO] train episode 1415: reward = -158.00, steps = 159
23:02:56 [INFO] train episode 1416: reward = -145.00, steps = 146
23:02:56 [INFO] train episode 1417: reward = -275.00, steps = 276
23:02:56 [INFO] train episode 1418: reward = -216.00, steps = 217
23:02:57 [INFO] train episode 1419: reward = -119.00, steps = 120
23:02:57 [INFO] train episode 1420: reward = -166.00, steps = 167
23:02:57 [INFO] train episode 1421: reward = -133.00, steps = 134
23:02:57 [INFO] train episode 1422: reward = -164.00, steps = 165
23:02:57 [INFO] train episode 1423: reward = -158.00, steps = 159
23:02:57 [INFO] train episode 1424: reward = -162.00, steps = 163
23:02:58 [INFO] train episode 1425: reward = -128.00, steps = 129
23:02:58 [INFO] train episode 1426: reward = -161.00, steps = 162
23:02:58 [INFO] train episode 1427: reward = -167.00, steps = 168
23:02:58 [INFO] train episode 1428: reward = -166.00, steps = 167
23:02:58 [INFO] train episode 1429: reward = -139.00, steps = 140
23:02:59 [INFO] train episode 1430: reward = -130.00, steps = 131
23:02:59 [INFO] train episode 1431: reward = -149.00, steps = 150
23:02:59 [INFO] train episode 1432: reward = -144.00, steps = 145
23:02:59 [INFO] train episode 1433: reward = -163.00, steps = 164
23:02:59 [INFO] train episode 1434: reward = -130.00, steps = 131
23:02:59 [INFO] train episode 1435: reward = -157.00, steps = 158
23:03:00 [INFO] train episode 1436: reward = -166.00, steps = 167
23:03:00 [INFO] train episode 1437: reward = -219.00, steps = 220
23:03:00 [INFO] train episode 1438: reward = -157.00, steps = 158
23:03:00 [INFO] train episode 1439: reward = -147.00, steps = 148
23:03:00 [INFO] train episode 1440: reward = -145.00, steps = 146
23:03:00 [INFO] train episode 1441: reward = -125.00, steps = 126
23:03:01 [INFO] train episode 1442: reward = -145.00, steps = 146
23:03:01 [INFO] train episode 1443: reward = -168.00, steps = 169
23:03:01 [INFO] train episode 1444: reward = -134.00, steps = 135
23:03:01 [INFO] train episode 1445: reward = -170.00, steps = 171
23:03:01 [INFO] train episode 1446: reward = -133.00, steps = 134
23:03:01 [INFO] train episode 1447: reward = -144.00, steps = 145
23:03:02 [INFO] train episode 1448: reward = -164.00, steps = 165
23:03:02 [INFO] train episode 1449: reward = -226.00, steps = 227
23:03:02 [INFO] train episode 1450: reward = -144.00, steps = 145
23:03:02 [INFO] train episode 1451: reward = -209.00, steps = 210
23:03:02 [INFO] train episode 1452: reward = -139.00, steps = 140
23:03:03 [INFO] train episode 1453: reward = -146.00, steps = 147
23:03:03 [INFO] train episode 1454: reward = -207.00, steps = 208
23:03:03 [INFO] train episode 1455: reward = -176.00, steps = 177
23:03:03 [INFO] train episode 1456: reward = -129.00, steps = 130
23:03:03 [INFO] train episode 1457: reward = -120.00, steps = 121
23:03:03 [INFO] train episode 1458: reward = -159.00, steps = 160
23:03:03 [INFO] train episode 1459: reward = -137.00, steps = 138
23:03:04 [INFO] train episode 1460: reward = -166.00, steps = 167
23:03:04 [INFO] train episode 1461: reward = -148.00, steps = 149
23:03:04 [INFO] train episode 1462: reward = -138.00, steps = 139
23:03:04 [INFO] train episode 1463: reward = -231.00, steps = 232
23:03:04 [INFO] train episode 1464: reward = -154.00, steps = 155
23:03:05 [INFO] train episode 1465: reward = -155.00, steps = 156
23:03:05 [INFO] train episode 1466: reward = -158.00, steps = 159
23:03:05 [INFO] train episode 1467: reward = -153.00, steps = 154
23:03:05 [INFO] train episode 1468: reward = -128.00, steps = 129
23:03:05 [INFO] train episode 1469: reward = -150.00, steps = 151
23:03:05 [INFO] train episode 1470: reward = -140.00, steps = 141
23:03:05 [INFO] train episode 1471: reward = -152.00, steps = 153
23:03:06 [INFO] train episode 1472: reward = -158.00, steps = 159
23:03:06 [INFO] train episode 1473: reward = -233.00, steps = 234
23:03:06 [INFO] train episode 1474: reward = -246.00, steps = 247
23:03:06 [INFO] train episode 1475: reward = -149.00, steps = 150
23:03:06 [INFO] train episode 1476: reward = -136.00, steps = 137
23:03:06 [INFO] train episode 1477: reward = -142.00, steps = 143
23:03:07 [INFO] train episode 1478: reward = -176.00, steps = 177
23:03:07 [INFO] train episode 1479: reward = -134.00, steps = 135
23:03:07 [INFO] train episode 1480: reward = -223.00, steps = 224
23:03:07 [INFO] train episode 1481: reward = -189.00, steps = 190
23:03:07 [INFO] train episode 1482: reward = -157.00, steps = 158
23:03:08 [INFO] train episode 1483: reward = -140.00, steps = 141
23:03:08 [INFO] train episode 1484: reward = -237.00, steps = 238
23:03:08 [INFO] train episode 1485: reward = -149.00, steps = 150
23:03:08 [INFO] train episode 1486: reward = -162.00, steps = 163
23:03:08 [INFO] train episode 1487: reward = -179.00, steps = 180
23:03:08 [INFO] train episode 1488: reward = -142.00, steps = 143
23:03:09 [INFO] train episode 1489: reward = -154.00, steps = 155
23:03:09 [INFO] train episode 1490: reward = -135.00, steps = 136
23:03:09 [INFO] train episode 1491: reward = -165.00, steps = 166
23:03:09 [INFO] train episode 1492: reward = -135.00, steps = 136
23:03:09 [INFO] train episode 1493: reward = -152.00, steps = 153
23:03:09 [INFO] train episode 1494: reward = -149.00, steps = 150
23:03:10 [INFO] train episode 1495: reward = -162.00, steps = 163
23:03:10 [INFO] train episode 1496: reward = -152.00, steps = 153
23:03:10 [INFO] train episode 1497: reward = -160.00, steps = 161
23:03:10 [INFO] train episode 1498: reward = -143.00, steps = 144
23:03:10 [INFO] train episode 1499: reward = -171.00, steps = 172
23:03:10 [INFO] train episode 1500: reward = -141.00, steps = 142
23:03:11 [INFO] train episode 1501: reward = -174.00, steps = 175
23:03:11 [INFO] train episode 1502: reward = -168.00, steps = 169
23:03:11 [INFO] train episode 1503: reward = -149.00, steps = 150
23:03:11 [INFO] train episode 1504: reward = -141.00, steps = 142
23:03:11 [INFO] train episode 1505: reward = -108.00, steps = 109
23:03:11 [INFO] train episode 1506: reward = -119.00, steps = 120
23:03:11 [INFO] train episode 1507: reward = -141.00, steps = 142
23:03:12 [INFO] train episode 1508: reward = -158.00, steps = 159
23:03:12 [INFO] train episode 1509: reward = -138.00, steps = 139
23:03:12 [INFO] train episode 1510: reward = -148.00, steps = 149
23:03:12 [INFO] train episode 1511: reward = -138.00, steps = 139
23:03:12 [INFO] train episode 1512: reward = -142.00, steps = 143
23:03:12 [INFO] train episode 1513: reward = -154.00, steps = 155
23:03:12 [INFO] train episode 1514: reward = -143.00, steps = 144
23:03:13 [INFO] train episode 1515: reward = -145.00, steps = 146
23:03:13 [INFO] train episode 1516: reward = -111.00, steps = 112
23:03:13 [INFO] train episode 1517: reward = -144.00, steps = 145
23:03:13 [INFO] train episode 1518: reward = -148.00, steps = 149
23:03:13 [INFO] train episode 1519: reward = -119.00, steps = 120
23:03:13 [INFO] train episode 1520: reward = -133.00, steps = 134
23:03:14 [INFO] train episode 1521: reward = -179.00, steps = 180
23:03:14 [INFO] train episode 1522: reward = -147.00, steps = 148
23:03:14 [INFO] train episode 1523: reward = -197.00, steps = 198
23:03:14 [INFO] train episode 1524: reward = -146.00, steps = 147
23:03:14 [INFO] train episode 1525: reward = -143.00, steps = 144
23:03:14 [INFO] train episode 1526: reward = -169.00, steps = 170
23:03:15 [INFO] train episode 1527: reward = -155.00, steps = 156
23:03:15 [INFO] train episode 1528: reward = -157.00, steps = 158
23:03:15 [INFO] train episode 1529: reward = -153.00, steps = 154
23:03:15 [INFO] train episode 1530: reward = -184.00, steps = 185
23:03:15 [INFO] train episode 1531: reward = -121.00, steps = 122
23:03:15 [INFO] train episode 1532: reward = -174.00, steps = 175
23:03:16 [INFO] train episode 1533: reward = -148.00, steps = 149
23:03:16 [INFO] train episode 1534: reward = -185.00, steps = 186
23:03:16 [INFO] train episode 1535: reward = -150.00, steps = 151
23:03:16 [INFO] train episode 1536: reward = -142.00, steps = 143
23:03:16 [INFO] train episode 1537: reward = -153.00, steps = 154
23:03:16 [INFO] train episode 1538: reward = -119.00, steps = 120
23:03:17 [INFO] train episode 1539: reward = -145.00, steps = 146
23:03:17 [INFO] train episode 1540: reward = -163.00, steps = 164
23:03:17 [INFO] train episode 1541: reward = -166.00, steps = 167
23:03:17 [INFO] train episode 1542: reward = -146.00, steps = 147
23:03:17 [INFO] train episode 1543: reward = -140.00, steps = 141
23:03:17 [INFO] train episode 1544: reward = -170.00, steps = 171
23:03:18 [INFO] train episode 1545: reward = -115.00, steps = 116
23:03:18 [INFO] train episode 1546: reward = -148.00, steps = 149
23:03:18 [INFO] train episode 1547: reward = -198.00, steps = 199
23:03:18 [INFO] train episode 1548: reward = -162.00, steps = 163
23:03:18 [INFO] train episode 1549: reward = -185.00, steps = 186
23:03:18 [INFO] train episode 1550: reward = -172.00, steps = 173
23:03:19 [INFO] train episode 1551: reward = -159.00, steps = 160
23:03:19 [INFO] train episode 1552: reward = -197.00, steps = 198
23:03:19 [INFO] train episode 1553: reward = -126.00, steps = 127
23:03:19 [INFO] train episode 1554: reward = -126.00, steps = 127
23:03:19 [INFO] train episode 1555: reward = -312.00, steps = 313
23:03:19 [INFO] train episode 1556: reward = -148.00, steps = 149
23:03:20 [INFO] train episode 1557: reward = -133.00, steps = 134
23:03:20 [INFO] train episode 1558: reward = -169.00, steps = 170
23:03:20 [INFO] train episode 1559: reward = -134.00, steps = 135
23:03:20 [INFO] train episode 1560: reward = -171.00, steps = 172
23:03:20 [INFO] train episode 1561: reward = -162.00, steps = 163
23:03:21 [INFO] train episode 1562: reward = -161.00, steps = 162
23:03:21 [INFO] train episode 1563: reward = -222.00, steps = 223
23:03:21 [INFO] train episode 1564: reward = -153.00, steps = 154
23:03:21 [INFO] train episode 1565: reward = -147.00, steps = 148
23:03:21 [INFO] train episode 1566: reward = -137.00, steps = 138
23:03:21 [INFO] train episode 1567: reward = -128.00, steps = 129
23:03:22 [INFO] train episode 1568: reward = -127.00, steps = 128
23:03:22 [INFO] train episode 1569: reward = -134.00, steps = 135
23:03:22 [INFO] train episode 1570: reward = -197.00, steps = 198
23:03:22 [INFO] train episode 1571: reward = -174.00, steps = 175
23:03:22 [INFO] train episode 1572: reward = -169.00, steps = 170
23:03:22 [INFO] train episode 1573: reward = -166.00, steps = 167
23:03:23 [INFO] train episode 1574: reward = -149.00, steps = 150
23:03:23 [INFO] train episode 1575: reward = -120.00, steps = 121
23:03:23 [INFO] train episode 1576: reward = -180.00, steps = 181
23:03:23 [INFO] train episode 1577: reward = -202.00, steps = 203
23:03:23 [INFO] train episode 1578: reward = -126.00, steps = 127
23:03:23 [INFO] train episode 1579: reward = -161.00, steps = 162
23:03:24 [INFO] train episode 1580: reward = -131.00, steps = 132
23:03:24 [INFO] train episode 1581: reward = -133.00, steps = 134
23:03:24 [INFO] train episode 1582: reward = -208.00, steps = 209
23:03:24 [INFO] train episode 1583: reward = -141.00, steps = 142
23:03:24 [INFO] train episode 1584: reward = -141.00, steps = 142
23:03:24 [INFO] train episode 1585: reward = -110.00, steps = 111
23:03:24 [INFO] train episode 1586: reward = -152.00, steps = 153
23:03:25 [INFO] train episode 1587: reward = -111.00, steps = 112
23:03:25 [INFO] train episode 1588: reward = -174.00, steps = 175
23:03:25 [INFO] train episode 1589: reward = -140.00, steps = 141
23:03:25 [INFO] train episode 1590: reward = -123.00, steps = 124
23:03:25 [INFO] train episode 1591: reward = -141.00, steps = 142
23:03:25 [INFO] train episode 1592: reward = -112.00, steps = 113
23:03:25 [INFO] train episode 1593: reward = -140.00, steps = 141
23:03:26 [INFO] train episode 1594: reward = -143.00, steps = 144
23:03:26 [INFO] train episode 1595: reward = -154.00, steps = 155
23:03:26 [INFO] train episode 1596: reward = -168.00, steps = 169
23:03:26 [INFO] train episode 1597: reward = -160.00, steps = 161
23:03:26 [INFO] train episode 1598: reward = -154.00, steps = 155
23:03:26 [INFO] train episode 1599: reward = -131.00, steps = 132
23:03:27 [INFO] train episode 1600: reward = -167.00, steps = 168
23:03:27 [INFO] train episode 1601: reward = -145.00, steps = 146
23:03:27 [INFO] train episode 1602: reward = -160.00, steps = 161
23:03:27 [INFO] train episode 1603: reward = -124.00, steps = 125
23:03:27 [INFO] train episode 1604: reward = -122.00, steps = 123
23:03:27 [INFO] train episode 1605: reward = -165.00, steps = 166
23:03:28 [INFO] train episode 1606: reward = -173.00, steps = 174
23:03:28 [INFO] train episode 1607: reward = -155.00, steps = 156
23:03:28 [INFO] train episode 1608: reward = -171.00, steps = 172
23:03:28 [INFO] train episode 1609: reward = -134.00, steps = 135
23:03:28 [INFO] train episode 1610: reward = -174.00, steps = 175
23:03:28 [INFO] train episode 1611: reward = -170.00, steps = 171
23:03:29 [INFO] train episode 1612: reward = -180.00, steps = 181
23:03:29 [INFO] train episode 1613: reward = -177.00, steps = 178
23:03:29 [INFO] train episode 1614: reward = -136.00, steps = 137
23:03:29 [INFO] train episode 1615: reward = -125.00, steps = 126
23:03:29 [INFO] train episode 1616: reward = -112.00, steps = 113
23:03:29 [INFO] train episode 1617: reward = -128.00, steps = 129
23:03:29 [INFO] train episode 1618: reward = -112.00, steps = 113
23:03:30 [INFO] train episode 1619: reward = -123.00, steps = 124
23:03:30 [INFO] train episode 1620: reward = -133.00, steps = 134
23:03:30 [INFO] train episode 1621: reward = -242.00, steps = 243
23:03:30 [INFO] train episode 1622: reward = -184.00, steps = 185
23:03:30 [INFO] train episode 1623: reward = -170.00, steps = 171
23:03:30 [INFO] train episode 1624: reward = -161.00, steps = 162
23:03:31 [INFO] train episode 1625: reward = -132.00, steps = 133
23:03:31 [INFO] train episode 1626: reward = -127.00, steps = 128
23:03:31 [INFO] train episode 1627: reward = -137.00, steps = 138
23:03:31 [INFO] train episode 1628: reward = -140.00, steps = 141
23:03:31 [INFO] train episode 1629: reward = -115.00, steps = 116
23:03:31 [INFO] train episode 1630: reward = -134.00, steps = 135
23:03:31 [INFO] train episode 1631: reward = -118.00, steps = 119
23:03:31 [INFO] train episode 1632: reward = -135.00, steps = 136
23:03:32 [INFO] train episode 1633: reward = -140.00, steps = 141
23:03:32 [INFO] train episode 1634: reward = -132.00, steps = 133
23:03:32 [INFO] train episode 1635: reward = -136.00, steps = 137
23:03:32 [INFO] train episode 1636: reward = -138.00, steps = 139
23:03:32 [INFO] train episode 1637: reward = -142.00, steps = 143
23:03:32 [INFO] train episode 1638: reward = -127.00, steps = 128
23:03:32 [INFO] train episode 1639: reward = -121.00, steps = 122
23:03:33 [INFO] train episode 1640: reward = -112.00, steps = 113
23:03:33 [INFO] train episode 1641: reward = -152.00, steps = 153
23:03:33 [INFO] train episode 1642: reward = -192.00, steps = 193
23:03:33 [INFO] train episode 1643: reward = -162.00, steps = 163
23:03:33 [INFO] train episode 1644: reward = -141.00, steps = 142
23:03:33 [INFO] train episode 1645: reward = -161.00, steps = 162
23:03:34 [INFO] train episode 1646: reward = -226.00, steps = 227
23:03:34 [INFO] train episode 1647: reward = -141.00, steps = 142
23:03:34 [INFO] train episode 1648: reward = -143.00, steps = 144
23:03:34 [INFO] train episode 1649: reward = -117.00, steps = 118
23:03:34 [INFO] train episode 1650: reward = -141.00, steps = 142
23:03:34 [INFO] train episode 1651: reward = -112.00, steps = 113
23:03:35 [INFO] train episode 1652: reward = -122.00, steps = 123
23:03:35 [INFO] train episode 1653: reward = -152.00, steps = 153
23:03:35 [INFO] train episode 1654: reward = -139.00, steps = 140
23:03:35 [INFO] train episode 1655: reward = -259.00, steps = 260
23:03:35 [INFO] train episode 1656: reward = -128.00, steps = 129
23:03:35 [INFO] train episode 1657: reward = -149.00, steps = 150
23:03:36 [INFO] train episode 1658: reward = -114.00, steps = 115
23:03:36 [INFO] train episode 1659: reward = -138.00, steps = 139
23:03:36 [INFO] train episode 1660: reward = -153.00, steps = 154
23:03:36 [INFO] train episode 1661: reward = -127.00, steps = 128
23:03:36 [INFO] train episode 1662: reward = -117.00, steps = 118
23:03:36 [INFO] train episode 1663: reward = -133.00, steps = 134
23:03:36 [INFO] train episode 1664: reward = -137.00, steps = 138
23:03:37 [INFO] train episode 1665: reward = -155.00, steps = 156
23:03:37 [INFO] train episode 1666: reward = -132.00, steps = 133
23:03:37 [INFO] train episode 1667: reward = -146.00, steps = 147
23:03:37 [INFO] train episode 1668: reward = -139.00, steps = 140
23:03:37 [INFO] train episode 1669: reward = -219.00, steps = 220
23:03:37 [INFO] train episode 1670: reward = -193.00, steps = 194
23:03:37 [INFO] train episode 1671: reward = -143.00, steps = 144
23:03:38 [INFO] train episode 1672: reward = -153.00, steps = 154
23:03:38 [INFO] train episode 1673: reward = -130.00, steps = 131
23:03:38 [INFO] train episode 1674: reward = -149.00, steps = 150
23:03:38 [INFO] train episode 1675: reward = -189.00, steps = 190
23:03:38 [INFO] train episode 1676: reward = -161.00, steps = 162
23:03:38 [INFO] train episode 1677: reward = -145.00, steps = 146
23:03:39 [INFO] train episode 1678: reward = -123.00, steps = 124
23:03:39 [INFO] train episode 1679: reward = -168.00, steps = 169
23:03:39 [INFO] train episode 1680: reward = -140.00, steps = 141
23:03:39 [INFO] train episode 1681: reward = -137.00, steps = 138
23:03:39 [INFO] train episode 1682: reward = -117.00, steps = 118
23:03:39 [INFO] train episode 1683: reward = -167.00, steps = 168
23:03:39 [INFO] train episode 1684: reward = -129.00, steps = 130
23:03:40 [INFO] train episode 1685: reward = -168.00, steps = 169
23:03:40 [INFO] train episode 1686: reward = -129.00, steps = 130
23:03:40 [INFO] train episode 1687: reward = -151.00, steps = 152
23:03:40 [INFO] train episode 1688: reward = -147.00, steps = 148
23:03:40 [INFO] train episode 1689: reward = -104.00, steps = 105
23:03:40 [INFO] train episode 1690: reward = -138.00, steps = 139
23:03:40 [INFO] train episode 1691: reward = -193.00, steps = 194
23:03:41 [INFO] train episode 1692: reward = -122.00, steps = 123
23:03:41 [INFO] train episode 1693: reward = -155.00, steps = 156
23:03:41 [INFO] train episode 1694: reward = -135.00, steps = 136
23:03:41 [INFO] train episode 1695: reward = -154.00, steps = 155
23:03:41 [INFO] train episode 1696: reward = -165.00, steps = 166
23:03:41 [INFO] train episode 1697: reward = -128.00, steps = 129
23:03:42 [INFO] train episode 1698: reward = -157.00, steps = 158
23:03:42 [INFO] train episode 1699: reward = -155.00, steps = 156
23:03:42 [INFO] train episode 1700: reward = -169.00, steps = 170
23:03:42 [INFO] train episode 1701: reward = -216.00, steps = 217
23:03:42 [INFO] train episode 1702: reward = -135.00, steps = 136
23:03:42 [INFO] train episode 1703: reward = -130.00, steps = 131
23:03:43 [INFO] train episode 1704: reward = -124.00, steps = 125
23:03:43 [INFO] train episode 1705: reward = -128.00, steps = 129
23:03:43 [INFO] train episode 1706: reward = -129.00, steps = 130
23:03:43 [INFO] train episode 1707: reward = -148.00, steps = 149
23:03:43 [INFO] train episode 1708: reward = -168.00, steps = 169
23:03:43 [INFO] train episode 1709: reward = -158.00, steps = 159
23:03:43 [INFO] train episode 1710: reward = -116.00, steps = 117
23:03:44 [INFO] train episode 1711: reward = -140.00, steps = 141
23:03:44 [INFO] train episode 1712: reward = -134.00, steps = 135
23:03:44 [INFO] train episode 1713: reward = -118.00, steps = 119
23:03:44 [INFO] train episode 1714: reward = -133.00, steps = 134
23:03:44 [INFO] train episode 1715: reward = -182.00, steps = 183
23:03:44 [INFO] train episode 1716: reward = -183.00, steps = 184
23:03:45 [INFO] train episode 1717: reward = -185.00, steps = 186
23:03:45 [INFO] train episode 1718: reward = -158.00, steps = 159
23:03:45 [INFO] train episode 1719: reward = -136.00, steps = 137
23:03:45 [INFO] train episode 1720: reward = -165.00, steps = 166
23:03:45 [INFO] train episode 1721: reward = -133.00, steps = 134
23:03:45 [INFO] train episode 1722: reward = -118.00, steps = 119
23:03:46 [INFO] train episode 1723: reward = -136.00, steps = 137
23:03:46 [INFO] train episode 1724: reward = -119.00, steps = 120
23:03:46 [INFO] train episode 1725: reward = -121.00, steps = 122
23:03:46 [INFO] train episode 1726: reward = -164.00, steps = 165
23:03:46 [INFO] train episode 1727: reward = -129.00, steps = 130
23:03:46 [INFO] train episode 1728: reward = -148.00, steps = 149
23:03:47 [INFO] train episode 1729: reward = -165.00, steps = 166
23:03:47 [INFO] train episode 1730: reward = -181.00, steps = 182
23:03:47 [INFO] train episode 1731: reward = -171.00, steps = 172
23:03:47 [INFO] train episode 1732: reward = -131.00, steps = 132
23:03:47 [INFO] train episode 1733: reward = -151.00, steps = 152
23:03:47 [INFO] train episode 1734: reward = -273.00, steps = 274
23:03:48 [INFO] train episode 1735: reward = -120.00, steps = 121
23:03:48 [INFO] train episode 1736: reward = -159.00, steps = 160
23:03:48 [INFO] train episode 1737: reward = -140.00, steps = 141
23:03:48 [INFO] train episode 1738: reward = -148.00, steps = 149
23:03:48 [INFO] train episode 1739: reward = -118.00, steps = 119
23:03:48 [INFO] train episode 1740: reward = -157.00, steps = 158
23:03:48 [INFO] train episode 1741: reward = -131.00, steps = 132
23:03:49 [INFO] train episode 1742: reward = -124.00, steps = 125
23:03:49 [INFO] train episode 1743: reward = -103.00, steps = 104
23:03:49 [INFO] train episode 1744: reward = -153.00, steps = 154
23:03:49 [INFO] train episode 1745: reward = -209.00, steps = 210
23:03:49 [INFO] train episode 1746: reward = -135.00, steps = 136
23:03:49 [INFO] train episode 1747: reward = -127.00, steps = 128
23:03:50 [INFO] train episode 1748: reward = -216.00, steps = 217
23:03:50 [INFO] train episode 1749: reward = -141.00, steps = 142
23:03:50 [INFO] train episode 1750: reward = -144.00, steps = 145
23:03:50 [INFO] train episode 1751: reward = -118.00, steps = 119
23:03:50 [INFO] train episode 1752: reward = -112.00, steps = 113
23:03:50 [INFO] train episode 1753: reward = -112.00, steps = 113
23:03:50 [INFO] train episode 1754: reward = -195.00, steps = 196
23:03:51 [INFO] train episode 1755: reward = -124.00, steps = 125
23:03:51 [INFO] train episode 1756: reward = -146.00, steps = 147
23:03:51 [INFO] train episode 1757: reward = -119.00, steps = 120
23:03:51 [INFO] train episode 1758: reward = -137.00, steps = 138
23:03:51 [INFO] train episode 1759: reward = -119.00, steps = 120
23:03:51 [INFO] train episode 1760: reward = -159.00, steps = 160
23:03:51 [INFO] train episode 1761: reward = -135.00, steps = 136
23:03:51 [INFO] train episode 1762: reward = -99.00, steps = 100
23:03:52 [INFO] train episode 1763: reward = -141.00, steps = 142
23:03:52 [INFO] train episode 1764: reward = -177.00, steps = 178
23:03:52 [INFO] train episode 1765: reward = -130.00, steps = 131
23:03:52 [INFO] train episode 1766: reward = -103.00, steps = 104
23:03:52 [INFO] train episode 1767: reward = -117.00, steps = 118
23:03:52 [INFO] train episode 1768: reward = -160.00, steps = 161
23:03:52 [INFO] train episode 1769: reward = -128.00, steps = 129
23:03:53 [INFO] train episode 1770: reward = -104.00, steps = 105
23:03:53 [INFO] train episode 1771: reward = -176.00, steps = 177
23:03:53 [INFO] train episode 1772: reward = -115.00, steps = 116
23:03:53 [INFO] train episode 1773: reward = -115.00, steps = 116
23:03:53 [INFO] train episode 1774: reward = -140.00, steps = 141
23:03:53 [INFO] train episode 1775: reward = -129.00, steps = 130
23:03:53 [INFO] train episode 1776: reward = -120.00, steps = 121
23:03:54 [INFO] train episode 1777: reward = -120.00, steps = 121
23:03:54 [INFO] train episode 1778: reward = -112.00, steps = 113
23:03:54 [INFO] train episode 1779: reward = -116.00, steps = 117
23:03:54 [INFO] train episode 1780: reward = -120.00, steps = 121
23:03:54 [INFO] train episode 1781: reward = -205.00, steps = 206
23:03:54 [INFO] train episode 1782: reward = -112.00, steps = 113
23:03:54 [INFO] train episode 1783: reward = -128.00, steps = 129
23:03:55 [INFO] train episode 1784: reward = -137.00, steps = 138
23:03:55 [INFO] train episode 1785: reward = -121.00, steps = 122
23:03:55 [INFO] train episode 1786: reward = -138.00, steps = 139
23:03:55 [INFO] train episode 1787: reward = -161.00, steps = 162
23:03:55 [INFO] train episode 1788: reward = -129.00, steps = 130
23:03:55 [INFO] train episode 1789: reward = -105.00, steps = 106
23:03:55 [INFO] train episode 1790: reward = -121.00, steps = 122
23:03:55 [INFO] train episode 1791: reward = -113.00, steps = 114
23:03:56 [INFO] train episode 1792: reward = -151.00, steps = 152
23:03:56 [INFO] train episode 1793: reward = -127.00, steps = 128
23:03:56 [INFO] train episode 1794: reward = -147.00, steps = 148
23:03:56 [INFO] train episode 1795: reward = -119.00, steps = 120
23:03:56 [INFO] train episode 1796: reward = -111.00, steps = 112
23:03:56 [INFO] train episode 1797: reward = -89.00, steps = 90
23:03:56 [INFO] train episode 1798: reward = -96.00, steps = 97
23:03:56 [INFO] ==== test ====
23:03:56 [INFO] test episode 0: reward = -125.00, steps = 126
23:03:57 [INFO] test episode 1: reward = -161.00, steps = 162
23:03:57 [INFO] test episode 2: reward = -103.00, steps = 104
23:03:57 [INFO] test episode 3: reward = -93.00, steps = 94
23:03:57 [INFO] test episode 4: reward = -114.00, steps = 115
23:03:57 [INFO] test episode 5: reward = -141.00, steps = 142
23:03:57 [INFO] test episode 6: reward = -114.00, steps = 115
23:03:57 [INFO] test episode 7: reward = -117.00, steps = 118
23:03:57 [INFO] test episode 8: reward = -85.00, steps = 86
23:03:57 [INFO] test episode 9: reward = -141.00, steps = 142
23:03:57 [INFO] test episode 10: reward = -144.00, steps = 145
23:03:57 [INFO] test episode 11: reward = -105.00, steps = 106
23:03:57 [INFO] test episode 12: reward = -105.00, steps = 106
23:03:58 [INFO] test episode 13: reward = -111.00, steps = 112
23:03:58 [INFO] test episode 14: reward = -113.00, steps = 114
23:03:58 [INFO] test episode 15: reward = -124.00, steps = 125
23:03:58 [INFO] test episode 16: reward = -111.00, steps = 112
23:03:58 [INFO] test episode 17: reward = -90.00, steps = 91
23:03:58 [INFO] test episode 18: reward = -96.00, steps = 97
23:03:58 [INFO] test episode 19: reward = -112.00, steps = 113
23:03:58 [INFO] test episode 20: reward = -262.00, steps = 263
23:03:58 [INFO] test episode 21: reward = -118.00, steps = 119
23:03:58 [INFO] test episode 22: reward = -162.00, steps = 163
23:03:59 [INFO] test episode 23: reward = -149.00, steps = 150
23:03:59 [INFO] test episode 24: reward = -134.00, steps = 135
23:03:59 [INFO] test episode 25: reward = -116.00, steps = 117
23:03:59 [INFO] test episode 26: reward = -104.00, steps = 105
23:03:59 [INFO] test episode 27: reward = -106.00, steps = 107
23:03:59 [INFO] test episode 28: reward = -121.00, steps = 122
23:03:59 [INFO] test episode 29: reward = -114.00, steps = 115
23:03:59 [INFO] test episode 30: reward = -142.00, steps = 143
23:03:59 [INFO] test episode 31: reward = -174.00, steps = 175
23:03:59 [INFO] test episode 32: reward = -108.00, steps = 109
23:03:59 [INFO] test episode 33: reward = -128.00, steps = 129
23:04:00 [INFO] test episode 34: reward = -112.00, steps = 113
23:04:00 [INFO] test episode 35: reward = -119.00, steps = 120
23:04:00 [INFO] test episode 36: reward = -123.00, steps = 124
23:04:00 [INFO] test episode 37: reward = -128.00, steps = 129
23:04:00 [INFO] test episode 38: reward = -103.00, steps = 104
23:04:00 [INFO] test episode 39: reward = -108.00, steps = 109
23:04:00 [INFO] test episode 40: reward = -94.00, steps = 95
23:04:00 [INFO] test episode 41: reward = -139.00, steps = 140
23:04:00 [INFO] test episode 42: reward = -142.00, steps = 143
23:04:00 [INFO] test episode 43: reward = -130.00, steps = 131
23:04:00 [INFO] test episode 44: reward = -94.00, steps = 95
23:04:01 [INFO] test episode 45: reward = -128.00, steps = 129
23:04:01 [INFO] test episode 46: reward = -143.00, steps = 144
23:04:01 [INFO] test episode 47: reward = -113.00, steps = 114
23:04:01 [INFO] test episode 48: reward = -134.00, steps = 135
23:04:01 [INFO] test episode 49: reward = -153.00, steps = 154
23:04:01 [INFO] test episode 50: reward = -101.00, steps = 102
23:04:01 [INFO] test episode 51: reward = -149.00, steps = 150
23:04:01 [INFO] test episode 52: reward = -140.00, steps = 141
23:04:01 [INFO] test episode 53: reward = -90.00, steps = 91
23:04:01 [INFO] test episode 54: reward = -129.00, steps = 130
23:04:01 [INFO] test episode 55: reward = -120.00, steps = 121
23:04:02 [INFO] test episode 56: reward = -128.00, steps = 129
23:04:02 [INFO] test episode 57: reward = -91.00, steps = 92
23:04:02 [INFO] test episode 58: reward = -103.00, steps = 104
23:04:02 [INFO] test episode 59: reward = -109.00, steps = 110
23:04:02 [INFO] test episode 60: reward = -102.00, steps = 103
23:04:02 [INFO] test episode 61: reward = -124.00, steps = 125
23:04:02 [INFO] test episode 62: reward = -124.00, steps = 125
23:04:02 [INFO] test episode 63: reward = -105.00, steps = 106
23:04:02 [INFO] test episode 64: reward = -125.00, steps = 126
23:04:02 [INFO] test episode 65: reward = -124.00, steps = 125
23:04:02 [INFO] test episode 66: reward = -96.00, steps = 97
23:04:02 [INFO] test episode 67: reward = -128.00, steps = 129
23:04:03 [INFO] test episode 68: reward = -121.00, steps = 122
23:04:03 [INFO] test episode 69: reward = -152.00, steps = 153
23:04:03 [INFO] test episode 70: reward = -151.00, steps = 152
23:04:03 [INFO] test episode 71: reward = -104.00, steps = 105
23:04:03 [INFO] test episode 72: reward = -123.00, steps = 124
23:04:03 [INFO] test episode 73: reward = -98.00, steps = 99
23:04:03 [INFO] test episode 74: reward = -95.00, steps = 96
23:04:03 [INFO] test episode 75: reward = -122.00, steps = 123
23:04:03 [INFO] test episode 76: reward = -135.00, steps = 136
23:04:03 [INFO] test episode 77: reward = -148.00, steps = 149
23:04:03 [INFO] test episode 78: reward = -124.00, steps = 125
23:04:04 [INFO] test episode 79: reward = -125.00, steps = 126
23:04:04 [INFO] test episode 80: reward = -174.00, steps = 175
23:04:04 [INFO] test episode 81: reward = -116.00, steps = 117
23:04:04 [INFO] test episode 82: reward = -103.00, steps = 104
23:04:04 [INFO] test episode 83: reward = -115.00, steps = 116
23:04:04 [INFO] test episode 84: reward = -110.00, steps = 111
23:04:04 [INFO] test episode 85: reward = -123.00, steps = 124
23:04:04 [INFO] test episode 86: reward = -139.00, steps = 140
23:04:04 [INFO] test episode 87: reward = -131.00, steps = 132
23:04:04 [INFO] test episode 88: reward = -101.00, steps = 102
23:04:04 [INFO] test episode 89: reward = -128.00, steps = 129
23:04:05 [INFO] test episode 90: reward = -141.00, steps = 142
23:04:05 [INFO] test episode 91: reward = -180.00, steps = 181
23:04:05 [INFO] test episode 92: reward = -120.00, steps = 121
23:04:05 [INFO] test episode 93: reward = -129.00, steps = 130
23:04:05 [INFO] test episode 94: reward = -114.00, steps = 115
23:04:05 [INFO] test episode 95: reward = -97.00, steps = 98
23:04:05 [INFO] test episode 96: reward = -138.00, steps = 139
23:04:05 [INFO] test episode 97: reward = -126.00, steps = 127
23:04:05 [INFO] test episode 98: reward = -169.00, steps = 170
23:04:05 [INFO] test episode 99: reward = -133.00, steps = 134
23:04:05 [INFO] average episode reward = -123.79 ± 24.41

env.close()