Use Soft Q Learning to Play LunarLander-v2

PyTorch version

%matplotlib inline

import sys
import logging
import itertools
import copy

import numpy as np
np.random.seed(0)
import pandas as pd
import gym
import matplotlib.pyplot as plt
import torch
torch.manual_seed(0)
import torch.nn as nn
import torch.optim as optim
import torch.distributions as distributions

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')

env = gym.make('LunarLander-v2')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

15:30:04 [INFO] env: <LunarLander<LunarLander-v2>>
15:30:04 [INFO] action_space: Discrete(4)
15:30:04 [INFO] observation_space: Box(-inf, inf, (8,), float32)
15:30:04 [INFO] reward_range: (-inf, inf)
15:30:04 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
15:30:04 [INFO] _max_episode_steps: 1000
15:30:04 [INFO] _elapsed_steps: None
15:30:04 [INFO] id: LunarLander-v2
15:30:04 [INFO] entry_point: gym.envs.box2d:LunarLander
15:30:04 [INFO] reward_threshold: 200
15:30:04 [INFO] nondeterministic: False
15:30:04 [INFO] max_episode_steps: 1000
15:30:04 [INFO] _kwargs: {}
15:30:04 [INFO] _env_name: LunarLander

class DQNReplayer:
    def __init__(self, capacity):
        self.memory = pd.DataFrame(index=range(capacity),
                columns=['state', 'action', 'reward', 'next_state', 'terminated'])
        self.i = 0
        self.count = 0
        self.capacity = capacity

    def store(self, *args):
        self.memory.loc[self.i] = np.asarray(args, dtype=object)
        self.i = (self.i + 1) % self.capacity
        self.count = min(self.count + 1, self.capacity)

    def sample(self, size):
        indices = np.random.choice(self.count, size=size)
        return (np.stack(self.memory.loc[indices, field]) for field in
                self.memory.columns)

class SQLAgent:
    def __init__(self, env):
        self.action_n = env.action_space.n
        self.gamma = 0.99

        self.replayer = DQNReplayer(10000)

        self.alpha = 0.02

        self.evaluate_net = self.build_net(
                input_size=env.observation_space.shape[0],
                hidden_sizes=[256, 256], output_size=self.action_n)
        self.optimizer = optim.Adam(self.evaluate_net.parameters(), lr=3e-4)
        self.loss = nn.MSELoss()

    def build_net(self, input_size, hidden_sizes, output_size):
        layers = []
        for input_size, output_size in zip(
                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):
            layers.append(nn.Linear(input_size, output_size))
            layers.append(nn.ReLU())
        layers = layers[:-1]
        model = nn.Sequential(*layers)
        return model

    def reset(self, mode=None):
        self.mode = mode
        if self.mode == 'train':
            self.trajectory = []
            self.target_net = copy.deepcopy(self.evaluate_net)

    def step(self, observation, reward, terminated):
        state_tensor = torch.as_tensor(observation,
                dtype=torch.float).squeeze(0)
        q_div_alpha_tensor = self.evaluate_net(state_tensor) / self.alpha
        v_div_alpha_tensor = torch.logsumexp(q_div_alpha_tensor, dim=-1,
                keepdim=True)
        prob_tensor = (q_div_alpha_tensor - v_div_alpha_tensor).exp()
        action_tensor = distributions.Categorical(prob_tensor).sample()
        action = action_tensor.item()
        if self.mode == 'train':
            self.trajectory += [observation, reward, terminated, action]
            if len(self.trajectory) >= 8:
                state, _, _, act, next_state, reward, terminated, _ = \
                        self.trajectory[-8:]
                self.replayer.store(state, act, reward, next_state, terminated)
            if self.replayer.count >= 500:
                self.learn()
        return action

    def close(self):
        pass

    def learn(self):
        # replay
        states, actions, rewards, next_states, terminateds = \
                self.replayer.sample(128)
        state_tensor = torch.as_tensor(states, dtype=torch.float)
        action_tensor = torch.as_tensor(actions, dtype=torch.long)
        reward_tensor = torch.as_tensor(rewards, dtype=torch.float)
        next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)
        terminated_tensor = torch.as_tensor(terminateds, dtype=torch.float)

        # update value net
        next_q_tensor = self.target_net(next_state_tensor)
        next_v_tensor = self.alpha * torch.logsumexp(next_q_tensor / self.alpha,
                dim=-1)
        target_tensor = reward_tensor + self.gamma * (1. - terminated_tensor) * \
                next_v_tensor
        pred_tensor = self.evaluate_net(state_tensor)
        q_tensor = pred_tensor.gather(1, action_tensor.unsqueeze(1)).squeeze(1)
        loss_tensor = self.loss(q_tensor, target_tensor.detach())
        self.optimizer.zero_grad()
        loss_tensor.backward()
        self.optimizer.step()


agent = SQLAgent(env)

def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== train ====')
episode_rewards = []
for episode in itertools.count():
    episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
            mode='train')
    episode_rewards.append(episode_reward)
    logging.info('train episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > 250:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

15:30:04 [INFO] ==== train ====
15:30:04 [INFO] train episode 0: reward = -461.18, steps = 83
15:30:04 [INFO] train episode 1: reward = -668.99, steps = 69
15:30:04 [INFO] train episode 2: reward = -458.32, steps = 73
15:30:04 [INFO] train episode 3: reward = -493.66, steps = 56
15:30:04 [INFO] train episode 4: reward = -546.61, steps = 76
15:30:04 [INFO] train episode 5: reward = -592.39, steps = 75
15:30:04 [INFO] train episode 6: reward = -591.07, steps = 68
15:30:05 [INFO] train episode 7: reward = 10.01, steps = 58
15:30:06 [INFO] train episode 8: reward = -101.41, steps = 53
15:30:06 [INFO] train episode 9: reward = -184.76, steps = 77
15:30:07 [INFO] train episode 10: reward = -127.31, steps = 61
15:30:08 [INFO] train episode 11: reward = -162.83, steps = 71
15:30:09 [INFO] train episode 12: reward = -151.20, steps = 79
15:30:09 [INFO] train episode 13: reward = -142.71, steps = 80
15:30:10 [INFO] train episode 14: reward = -159.48, steps = 66
15:30:11 [INFO] train episode 15: reward = -109.68, steps = 77
15:30:12 [INFO] train episode 16: reward = -162.97, steps = 68
15:30:13 [INFO] train episode 17: reward = -78.89, steps = 55
15:30:14 [INFO] train episode 18: reward = -856.41, steps = 154
15:30:16 [INFO] train episode 19: reward = -717.81, steps = 110
15:30:16 [INFO] train episode 20: reward = -109.94, steps = 63
15:30:17 [INFO] train episode 21: reward = -124.91, steps = 61
15:30:18 [INFO] train episode 22: reward = -260.72, steps = 100
15:30:19 [INFO] train episode 23: reward = -118.52, steps = 50
15:30:20 [INFO] train episode 24: reward = -44.96, steps = 89
15:30:21 [INFO] train episode 25: reward = -84.43, steps = 56
15:30:22 [INFO] train episode 26: reward = -74.78, steps = 87
15:30:24 [INFO] train episode 27: reward = -382.92, steps = 175
15:30:25 [INFO] train episode 28: reward = -179.34, steps = 84
15:30:26 [INFO] train episode 29: reward = -59.86, steps = 56
15:30:27 [INFO] train episode 30: reward = -65.31, steps = 94
15:30:28 [INFO] train episode 31: reward = -49.00, steps = 75
15:30:29 [INFO] train episode 32: reward = 42.47, steps = 88
15:30:30 [INFO] train episode 33: reward = -196.26, steps = 91
15:30:32 [INFO] train episode 34: reward = -1080.69, steps = 159
15:30:35 [INFO] train episode 35: reward = -346.48, steps = 190
15:30:37 [INFO] train episode 36: reward = -272.09, steps = 217
15:30:38 [INFO] train episode 37: reward = -108.51, steps = 102
15:30:39 [INFO] train episode 38: reward = -422.14, steps = 80
15:30:40 [INFO] train episode 39: reward = -469.50, steps = 95
15:30:42 [INFO] train episode 40: reward = -349.59, steps = 190
15:30:43 [INFO] train episode 41: reward = -212.93, steps = 104
15:30:45 [INFO] train episode 42: reward = -257.79, steps = 128
15:30:47 [INFO] train episode 43: reward = -293.18, steps = 215
15:30:49 [INFO] train episode 44: reward = -199.20, steps = 150
15:30:51 [INFO] train episode 45: reward = -223.86, steps = 158
15:30:52 [INFO] train episode 46: reward = -204.57, steps = 134
15:30:54 [INFO] train episode 47: reward = -198.53, steps = 174
15:30:56 [INFO] train episode 48: reward = -358.08, steps = 155
15:30:58 [INFO] train episode 49: reward = -216.80, steps = 155
15:30:59 [INFO] train episode 50: reward = -131.84, steps = 171
15:31:01 [INFO] train episode 51: reward = -224.78, steps = 128
15:31:03 [INFO] train episode 52: reward = -138.28, steps = 160
15:31:04 [INFO] train episode 53: reward = -94.56, steps = 150
15:31:06 [INFO] train episode 54: reward = -73.09, steps = 156
15:31:09 [INFO] train episode 55: reward = -381.66, steps = 242
15:31:10 [INFO] train episode 56: reward = -280.63, steps = 149
15:31:13 [INFO] train episode 57: reward = -262.54, steps = 236
15:31:15 [INFO] train episode 58: reward = -494.19, steps = 168
15:31:17 [INFO] train episode 59: reward = -168.34, steps = 149
15:31:19 [INFO] train episode 60: reward = -319.37, steps = 232
15:31:21 [INFO] train episode 61: reward = -172.28, steps = 137
15:31:23 [INFO] train episode 62: reward = -217.15, steps = 143
15:31:24 [INFO] train episode 63: reward = -216.08, steps = 154
15:31:30 [INFO] train episode 64: reward = -192.72, steps = 434
15:31:32 [INFO] train episode 65: reward = -186.88, steps = 149
15:31:33 [INFO] train episode 66: reward = -183.78, steps = 139
15:31:35 [INFO] train episode 67: reward = -352.77, steps = 167
15:31:37 [INFO] train episode 68: reward = 15.43, steps = 149
15:31:38 [INFO] train episode 69: reward = -253.02, steps = 124
15:31:42 [INFO] train episode 70: reward = -259.30, steps = 325
15:31:44 [INFO] train episode 71: reward = -118.72, steps = 189
15:31:47 [INFO] train episode 72: reward = -279.75, steps = 231
15:31:50 [INFO] train episode 73: reward = -291.15, steps = 217
15:31:51 [INFO] train episode 74: reward = -136.00, steps = 140
15:31:53 [INFO] train episode 75: reward = -189.93, steps = 153
15:31:55 [INFO] train episode 76: reward = -196.52, steps = 154
15:31:56 [INFO] train episode 77: reward = -101.84, steps = 160
15:31:58 [INFO] train episode 78: reward = -180.87, steps = 146
15:32:02 [INFO] train episode 79: reward = -271.79, steps = 283
15:32:05 [INFO] train episode 80: reward = -444.22, steps = 248
15:32:10 [INFO] train episode 81: reward = -197.07, steps = 343
15:32:12 [INFO] train episode 82: reward = -102.25, steps = 140
15:32:17 [INFO] train episode 83: reward = -220.68, steps = 253
15:32:18 [INFO] train episode 84: reward = -187.62, steps = 101
15:32:21 [INFO] train episode 85: reward = -233.20, steps = 206
15:32:26 [INFO] train episode 86: reward = -266.51, steps = 369
15:32:31 [INFO] train episode 87: reward = -232.73, steps = 340
15:32:35 [INFO] train episode 88: reward = -165.14, steps = 301
15:32:52 [INFO] train episode 89: reward = -167.00, steps = 1000
15:32:56 [INFO] train episode 90: reward = -237.51, steps = 219
15:33:13 [INFO] train episode 91: reward = -201.66, steps = 1000
15:33:15 [INFO] train episode 92: reward = -180.10, steps = 188
15:33:31 [INFO] train episode 93: reward = -148.77, steps = 1000
15:33:47 [INFO] train episode 94: reward = -150.70, steps = 1000
15:34:02 [INFO] train episode 95: reward = -181.40, steps = 1000
15:34:04 [INFO] train episode 96: reward = -100.86, steps = 141
15:34:08 [INFO] train episode 97: reward = -154.23, steps = 246
15:34:23 [INFO] train episode 98: reward = -161.37, steps = 1000
15:34:24 [INFO] train episode 99: reward = -291.60, steps = 95
15:34:40 [INFO] train episode 100: reward = -149.28, steps = 1000
15:34:56 [INFO] train episode 101: reward = -140.16, steps = 1000
15:35:12 [INFO] train episode 102: reward = -201.76, steps = 1000
15:35:29 [INFO] train episode 103: reward = -156.56, steps = 1000
15:35:47 [INFO] train episode 104: reward = -130.02, steps = 1000
15:36:03 [INFO] train episode 105: reward = -165.13, steps = 1000
15:36:21 [INFO] train episode 106: reward = -209.76, steps = 1000
15:36:36 [INFO] train episode 107: reward = -161.81, steps = 1000
15:36:37 [INFO] train episode 108: reward = -145.19, steps = 83
15:36:52 [INFO] train episode 109: reward = -179.72, steps = 1000
15:37:07 [INFO] train episode 110: reward = -192.56, steps = 1000
15:37:22 [INFO] train episode 111: reward = -202.50, steps = 1000
15:37:38 [INFO] train episode 112: reward = -167.27, steps = 1000
15:37:56 [INFO] train episode 113: reward = -178.42, steps = 1000
15:38:11 [INFO] train episode 114: reward = -175.40, steps = 1000
15:38:26 [INFO] train episode 115: reward = -205.94, steps = 1000
15:38:42 [INFO] train episode 116: reward = -165.85, steps = 1000
15:38:58 [INFO] train episode 117: reward = -183.95, steps = 1000
15:39:14 [INFO] train episode 118: reward = -201.14, steps = 1000
15:39:30 [INFO] train episode 119: reward = -155.56, steps = 1000
15:39:45 [INFO] train episode 120: reward = -139.84, steps = 1000
15:40:02 [INFO] train episode 121: reward = -141.59, steps = 1000
15:40:18 [INFO] train episode 122: reward = -167.62, steps = 1000
15:40:34 [INFO] train episode 123: reward = -148.34, steps = 1000
15:40:48 [INFO] train episode 124: reward = -98.38, steps = 1000
15:41:06 [INFO] train episode 125: reward = -173.27, steps = 1000
15:41:22 [INFO] train episode 126: reward = -134.38, steps = 1000
15:41:39 [INFO] train episode 127: reward = -123.90, steps = 1000
15:41:41 [INFO] train episode 128: reward = -260.99, steps = 135
15:41:58 [INFO] train episode 129: reward = -205.37, steps = 1000
15:42:15 [INFO] train episode 130: reward = -161.72, steps = 1000
15:42:31 [INFO] train episode 131: reward = -171.49, steps = 1000
15:42:47 [INFO] train episode 132: reward = -145.85, steps = 1000
15:43:03 [INFO] train episode 133: reward = -140.45, steps = 1000
15:43:18 [INFO] train episode 134: reward = -144.73, steps = 1000
15:43:37 [INFO] train episode 135: reward = -145.92, steps = 1000
15:43:54 [INFO] train episode 136: reward = -131.38, steps = 1000
15:44:13 [INFO] train episode 137: reward = -159.54, steps = 1000
15:44:30 [INFO] train episode 138: reward = -164.33, steps = 1000
15:44:51 [INFO] train episode 139: reward = -151.81, steps = 1000
15:45:10 [INFO] train episode 140: reward = -159.38, steps = 1000
15:45:29 [INFO] train episode 141: reward = -119.14, steps = 1000
15:45:46 [INFO] train episode 142: reward = -149.16, steps = 1000
15:46:03 [INFO] train episode 143: reward = -134.57, steps = 1000
15:46:08 [INFO] train episode 144: reward = -172.85, steps = 329
15:46:27 [INFO] train episode 145: reward = -146.75, steps = 1000
15:46:40 [INFO] train episode 146: reward = -228.53, steps = 671
15:46:57 [INFO] train episode 147: reward = -133.78, steps = 1000
15:47:17 [INFO] train episode 148: reward = -170.46, steps = 1000
15:47:34 [INFO] train episode 149: reward = -132.13, steps = 1000
15:47:51 [INFO] train episode 150: reward = -155.12, steps = 1000
15:48:09 [INFO] train episode 151: reward = -115.96, steps = 1000
15:48:26 [INFO] train episode 152: reward = -128.12, steps = 1000
15:48:46 [INFO] train episode 153: reward = -136.80, steps = 1000
15:49:06 [INFO] train episode 154: reward = -130.60, steps = 1000
15:49:24 [INFO] train episode 155: reward = -104.23, steps = 1000
15:49:51 [INFO] train episode 156: reward = -154.83, steps = 1000
15:50:10 [INFO] train episode 157: reward = -132.27, steps = 1000
15:50:14 [INFO] train episode 158: reward = -141.06, steps = 257
15:50:30 [INFO] train episode 159: reward = -129.47, steps = 1000
15:50:31 [INFO] train episode 160: reward = -299.50, steps = 80
15:50:48 [INFO] train episode 161: reward = -140.84, steps = 1000
15:51:00 [INFO] train episode 162: reward = -244.77, steps = 816
15:51:16 [INFO] train episode 163: reward = -91.54, steps = 1000
15:51:28 [INFO] train episode 164: reward = -195.96, steps = 743
15:51:30 [INFO] train episode 165: reward = -124.06, steps = 139
15:51:46 [INFO] train episode 166: reward = -132.26, steps = 1000
15:51:50 [INFO] train episode 167: reward = -77.08, steps = 289
15:51:53 [INFO] train episode 168: reward = -75.81, steps = 231
15:52:08 [INFO] train episode 169: reward = -118.95, steps = 1000
15:52:15 [INFO] train episode 170: reward = -234.10, steps = 453
15:52:32 [INFO] train episode 171: reward = -128.45, steps = 1000
15:52:41 [INFO] train episode 172: reward = -331.69, steps = 600
15:52:58 [INFO] train episode 173: reward = -73.57, steps = 1000
15:53:15 [INFO] train episode 174: reward = -114.12, steps = 1000
15:53:32 [INFO] train episode 175: reward = -160.87, steps = 1000
15:53:50 [INFO] train episode 176: reward = -134.43, steps = 1000
15:54:06 [INFO] train episode 177: reward = -116.93, steps = 1000
15:54:22 [INFO] train episode 178: reward = -178.51, steps = 1000
15:54:39 [INFO] train episode 179: reward = -84.95, steps = 1000
15:54:54 [INFO] train episode 180: reward = -117.83, steps = 1000
15:55:11 [INFO] train episode 181: reward = -127.32, steps = 1000
15:55:26 [INFO] train episode 182: reward = -105.13, steps = 1000
15:55:43 [INFO] train episode 183: reward = -117.50, steps = 1000
15:55:58 [INFO] train episode 184: reward = -107.08, steps = 1000
15:56:16 [INFO] train episode 185: reward = -111.60, steps = 1000
15:56:32 [INFO] train episode 186: reward = -101.00, steps = 1000
15:56:48 [INFO] train episode 187: reward = -106.90, steps = 1000
15:57:04 [INFO] train episode 188: reward = -116.72, steps = 1000
15:57:21 [INFO] train episode 189: reward = -145.83, steps = 1000
15:57:38 [INFO] train episode 190: reward = -94.56, steps = 1000
15:57:56 [INFO] train episode 191: reward = -158.11, steps = 1000
15:58:14 [INFO] train episode 192: reward = -159.91, steps = 1000
15:58:32 [INFO] train episode 193: reward = -119.21, steps = 1000
15:58:47 [INFO] train episode 194: reward = -115.40, steps = 1000
15:59:04 [INFO] train episode 195: reward = -68.14, steps = 1000
15:59:23 [INFO] train episode 196: reward = -132.07, steps = 1000
15:59:40 [INFO] train episode 197: reward = -144.48, steps = 1000
15:59:57 [INFO] train episode 198: reward = -116.21, steps = 1000
15:59:58 [INFO] train episode 199: reward = -102.59, steps = 58
16:00:14 [INFO] train episode 200: reward = -103.85, steps = 1000
16:00:31 [INFO] train episode 201: reward = -131.40, steps = 1000
16:00:52 [INFO] train episode 202: reward = -70.19, steps = 1000
16:01:09 [INFO] train episode 203: reward = -142.31, steps = 1000
16:01:25 [INFO] train episode 204: reward = -145.11, steps = 1000
16:01:41 [INFO] train episode 205: reward = -124.93, steps = 1000
16:01:56 [INFO] train episode 206: reward = -74.29, steps = 1000
16:02:12 [INFO] train episode 207: reward = -33.20, steps = 1000
16:02:28 [INFO] train episode 208: reward = -89.19, steps = 1000
16:02:44 [INFO] train episode 209: reward = -67.87, steps = 1000
16:03:00 [INFO] train episode 210: reward = -90.71, steps = 1000
16:03:16 [INFO] train episode 211: reward = -110.03, steps = 1000
16:03:32 [INFO] train episode 212: reward = -66.46, steps = 1000
16:03:49 [INFO] train episode 213: reward = -98.37, steps = 1000
16:04:05 [INFO] train episode 214: reward = -56.01, steps = 1000
16:04:21 [INFO] train episode 215: reward = -89.63, steps = 1000
16:04:38 [INFO] train episode 216: reward = -84.69, steps = 1000
16:04:54 [INFO] train episode 217: reward = -67.74, steps = 1000
16:05:11 [INFO] train episode 218: reward = -105.08, steps = 1000
16:05:12 [INFO] train episode 219: reward = -91.82, steps = 52
16:05:28 [INFO] train episode 220: reward = -55.65, steps = 1000
16:05:43 [INFO] train episode 221: reward = -86.73, steps = 1000
16:05:58 [INFO] train episode 222: reward = -45.91, steps = 1000
16:06:16 [INFO] train episode 223: reward = -49.99, steps = 1000
16:06:32 [INFO] train episode 224: reward = -32.55, steps = 1000
16:06:49 [INFO] train episode 225: reward = -78.87, steps = 1000
16:07:06 [INFO] train episode 226: reward = -44.40, steps = 1000
16:07:22 [INFO] train episode 227: reward = -54.29, steps = 1000
16:07:38 [INFO] train episode 228: reward = -82.28, steps = 1000
16:07:54 [INFO] train episode 229: reward = -68.36, steps = 1000
16:08:09 [INFO] train episode 230: reward = -26.31, steps = 1000
16:08:24 [INFO] train episode 231: reward = -86.26, steps = 1000
16:08:40 [INFO] train episode 232: reward = -46.15, steps = 1000
16:08:56 [INFO] train episode 233: reward = -155.39, steps = 1000
16:09:12 [INFO] train episode 234: reward = -41.64, steps = 1000
16:09:29 [INFO] train episode 235: reward = -88.05, steps = 1000
16:09:46 [INFO] train episode 236: reward = -87.17, steps = 1000
16:10:01 [INFO] train episode 237: reward = -168.74, steps = 991
16:10:14 [INFO] train episode 238: reward = -159.35, steps = 821
16:10:31 [INFO] train episode 239: reward = -89.67, steps = 1000
16:10:47 [INFO] train episode 240: reward = -229.81, steps = 944
16:11:05 [INFO] train episode 241: reward = -48.93, steps = 1000
16:11:20 [INFO] train episode 242: reward = -30.57, steps = 1000
16:11:38 [INFO] train episode 243: reward = -98.84, steps = 1000
16:11:54 [INFO] train episode 244: reward = -59.22, steps = 1000
16:12:11 [INFO] train episode 245: reward = -47.49, steps = 1000
16:12:29 [INFO] train episode 246: reward = -49.54, steps = 1000
16:12:45 [INFO] train episode 247: reward = -55.58, steps = 1000
16:13:01 [INFO] train episode 248: reward = -261.72, steps = 969
16:13:19 [INFO] train episode 249: reward = -40.42, steps = 1000
16:13:35 [INFO] train episode 250: reward = -32.94, steps = 1000
16:13:52 [INFO] train episode 251: reward = -24.56, steps = 1000
16:14:08 [INFO] train episode 252: reward = -16.88, steps = 1000
16:14:26 [INFO] train episode 253: reward = -56.63, steps = 1000
16:14:42 [INFO] train episode 254: reward = -38.38, steps = 1000
16:14:59 [INFO] train episode 255: reward = -54.97, steps = 1000
16:15:17 [INFO] train episode 256: reward = -79.20, steps = 1000
16:15:35 [INFO] train episode 257: reward = -76.28, steps = 1000
16:15:52 [INFO] train episode 258: reward = -59.60, steps = 1000
16:16:09 [INFO] train episode 259: reward = -35.59, steps = 1000
16:16:27 [INFO] train episode 260: reward = -51.68, steps = 1000
16:16:44 [INFO] train episode 261: reward = -51.66, steps = 1000
16:17:01 [INFO] train episode 262: reward = -32.21, steps = 1000
16:17:19 [INFO] train episode 263: reward = -74.43, steps = 1000
16:17:36 [INFO] train episode 264: reward = -60.47, steps = 1000
16:17:54 [INFO] train episode 265: reward = -34.44, steps = 1000
16:18:11 [INFO] train episode 266: reward = -17.53, steps = 1000
16:18:28 [INFO] train episode 267: reward = -50.44, steps = 1000
16:18:47 [INFO] train episode 268: reward = -31.32, steps = 1000
16:19:05 [INFO] train episode 269: reward = -74.84, steps = 1000
16:19:23 [INFO] train episode 270: reward = -71.76, steps = 1000
16:19:42 [INFO] train episode 271: reward = -26.28, steps = 1000
16:20:00 [INFO] train episode 272: reward = -78.65, steps = 1000
16:20:19 [INFO] train episode 273: reward = -54.83, steps = 1000
16:20:36 [INFO] train episode 274: reward = -20.61, steps = 1000
16:20:53 [INFO] train episode 275: reward = -57.88, steps = 1000
16:21:12 [INFO] train episode 276: reward = -44.64, steps = 1000
16:21:28 [INFO] train episode 277: reward = -27.04, steps = 1000
16:21:45 [INFO] train episode 278: reward = -39.71, steps = 1000
16:22:05 [INFO] train episode 279: reward = -55.80, steps = 1000
16:22:23 [INFO] train episode 280: reward = -67.25, steps = 1000
16:22:39 [INFO] train episode 281: reward = -33.43, steps = 1000
16:22:59 [INFO] train episode 282: reward = -36.89, steps = 1000
16:23:17 [INFO] train episode 283: reward = -109.18, steps = 1000
16:23:23 [INFO] train episode 284: reward = -637.89, steps = 368
16:23:41 [INFO] train episode 285: reward = -56.64, steps = 1000
16:23:58 [INFO] train episode 286: reward = -60.65, steps = 1000
16:24:14 [INFO] train episode 287: reward = -23.12, steps = 1000
16:24:31 [INFO] train episode 288: reward = -67.32, steps = 1000
16:24:46 [INFO] train episode 289: reward = -70.14, steps = 1000
16:25:03 [INFO] train episode 290: reward = -11.99, steps = 1000
16:25:21 [INFO] train episode 291: reward = -82.50, steps = 1000
16:25:38 [INFO] train episode 292: reward = -24.19, steps = 1000
16:25:55 [INFO] train episode 293: reward = -75.30, steps = 1000
16:26:11 [INFO] train episode 294: reward = -55.99, steps = 1000
16:26:27 [INFO] train episode 295: reward = 10.52, steps = 1000
16:26:43 [INFO] train episode 296: reward = -49.16, steps = 1000
16:27:02 [INFO] train episode 297: reward = -57.18, steps = 1000
16:27:18 [INFO] train episode 298: reward = -35.64, steps = 1000
16:27:35 [INFO] train episode 299: reward = -661.19, steps = 1000
16:27:52 [INFO] train episode 300: reward = -66.01, steps = 1000
16:27:52 [INFO] train episode 301: reward = -109.61, steps = 53
16:28:08 [INFO] train episode 302: reward = -184.58, steps = 1000
16:28:24 [INFO] train episode 303: reward = -61.92, steps = 1000
16:28:41 [INFO] train episode 304: reward = -71.44, steps = 1000
16:28:43 [INFO] train episode 305: reward = -105.97, steps = 180
16:29:00 [INFO] train episode 306: reward = -5.41, steps = 1000
16:29:16 [INFO] train episode 307: reward = -89.25, steps = 1000
16:29:33 [INFO] train episode 308: reward = -80.65, steps = 1000
16:29:50 [INFO] train episode 309: reward = -35.80, steps = 1000
16:30:07 [INFO] train episode 310: reward = -41.52, steps = 1000
16:30:23 [INFO] train episode 311: reward = -82.70, steps = 1000
16:30:25 [INFO] train episode 312: reward = -120.78, steps = 131
16:30:30 [INFO] train episode 313: reward = -174.65, steps = 297
16:30:32 [INFO] train episode 314: reward = -60.49, steps = 185
16:30:48 [INFO] train episode 315: reward = -33.68, steps = 1000
16:30:50 [INFO] train episode 316: reward = -327.84, steps = 130
16:30:52 [INFO] train episode 317: reward = -368.02, steps = 134
16:30:55 [INFO] train episode 318: reward = -374.26, steps = 207
16:30:57 [INFO] train episode 319: reward = -258.82, steps = 108
16:31:13 [INFO] train episode 320: reward = -31.46, steps = 1000
16:31:30 [INFO] train episode 321: reward = -45.66, steps = 1000
16:31:46 [INFO] train episode 322: reward = -34.57, steps = 1000
16:31:52 [INFO] train episode 323: reward = -209.25, steps = 403
16:32:08 [INFO] train episode 324: reward = -39.94, steps = 1000
16:32:11 [INFO] train episode 325: reward = -135.46, steps = 163
16:32:27 [INFO] train episode 326: reward = -10.92, steps = 1000
16:32:43 [INFO] train episode 327: reward = -45.30, steps = 1000
16:33:00 [INFO] train episode 328: reward = -51.49, steps = 1000
16:33:09 [INFO] train episode 329: reward = -22.21, steps = 524
16:33:25 [INFO] train episode 330: reward = -32.08, steps = 1000
16:33:32 [INFO] train episode 331: reward = -189.42, steps = 447
16:33:35 [INFO] train episode 332: reward = -78.06, steps = 209
16:33:42 [INFO] train episode 333: reward = -208.59, steps = 487
16:33:59 [INFO] train episode 334: reward = -12.92, steps = 1000
16:34:15 [INFO] train episode 335: reward = -26.31, steps = 1000
16:34:32 [INFO] train episode 336: reward = -78.34, steps = 1000
16:34:48 [INFO] train episode 337: reward = 21.48, steps = 1000
16:35:06 [INFO] train episode 338: reward = -58.45, steps = 1000
16:35:22 [INFO] train episode 339: reward = 11.60, steps = 1000
16:35:38 [INFO] train episode 340: reward = 10.61, steps = 1000
16:35:53 [INFO] train episode 341: reward = 3.54, steps = 1000
16:36:09 [INFO] train episode 342: reward = 15.60, steps = 1000
16:36:09 [INFO] train episode 343: reward = -119.59, steps = 52
16:36:24 [INFO] train episode 344: reward = -32.67, steps = 1000
16:36:40 [INFO] train episode 345: reward = 10.68, steps = 1000
16:36:57 [INFO] train episode 346: reward = -35.37, steps = 1000
16:37:12 [INFO] train episode 347: reward = 9.28, steps = 1000
16:37:28 [INFO] train episode 348: reward = -22.10, steps = 1000
16:37:44 [INFO] train episode 349: reward = 24.38, steps = 1000
16:38:00 [INFO] train episode 350: reward = 8.58, steps = 1000
16:38:17 [INFO] train episode 351: reward = 1.40, steps = 1000
16:38:35 [INFO] train episode 352: reward = -51.02, steps = 1000
16:38:52 [INFO] train episode 353: reward = -68.66, steps = 1000
16:39:08 [INFO] train episode 354: reward = 13.89, steps = 1000
16:39:24 [INFO] train episode 355: reward = 8.85, steps = 1000
16:39:40 [INFO] train episode 356: reward = -37.26, steps = 1000
16:39:56 [INFO] train episode 357: reward = 12.02, steps = 1000
16:40:12 [INFO] train episode 358: reward = -47.81, steps = 1000
16:40:28 [INFO] train episode 359: reward = -21.64, steps = 1000
16:40:45 [INFO] train episode 360: reward = 4.33, steps = 1000
16:41:02 [INFO] train episode 361: reward = -14.15, steps = 1000
16:41:18 [INFO] train episode 362: reward = -0.50, steps = 1000
16:41:35 [INFO] train episode 363: reward = -32.45, steps = 1000
16:41:52 [INFO] train episode 364: reward = -44.04, steps = 1000
16:42:08 [INFO] train episode 365: reward = -7.90, steps = 1000
16:42:26 [INFO] train episode 366: reward = -42.57, steps = 1000
16:42:42 [INFO] train episode 367: reward = -22.23, steps = 1000
16:43:00 [INFO] train episode 368: reward = -26.27, steps = 1000
16:43:16 [INFO] train episode 369: reward = -16.67, steps = 1000
16:43:34 [INFO] train episode 370: reward = -12.73, steps = 1000
16:43:51 [INFO] train episode 371: reward = -13.88, steps = 1000
16:44:08 [INFO] train episode 372: reward = -13.60, steps = 1000
16:44:24 [INFO] train episode 373: reward = 20.07, steps = 1000
16:44:43 [INFO] train episode 374: reward = -39.98, steps = 1000
16:44:59 [INFO] train episode 375: reward = 1.83, steps = 1000
16:45:16 [INFO] train episode 376: reward = 8.48, steps = 1000
16:45:32 [INFO] train episode 377: reward = -35.34, steps = 1000
16:45:49 [INFO] train episode 378: reward = -15.22, steps = 1000
16:46:05 [INFO] train episode 379: reward = -11.67, steps = 1000
16:46:07 [INFO] train episode 380: reward = -91.93, steps = 135
16:46:23 [INFO] train episode 381: reward = 3.62, steps = 1000
16:46:39 [INFO] train episode 382: reward = -17.76, steps = 1000
16:46:56 [INFO] train episode 383: reward = -45.99, steps = 1000
16:47:13 [INFO] train episode 384: reward = 14.77, steps = 1000
16:47:29 [INFO] train episode 385: reward = 7.12, steps = 1000
16:47:36 [INFO] train episode 386: reward = -497.05, steps = 434
16:47:52 [INFO] train episode 387: reward = -8.96, steps = 1000
16:48:10 [INFO] train episode 388: reward = -62.27, steps = 1000
16:48:26 [INFO] train episode 389: reward = -9.82, steps = 1000
16:48:43 [INFO] train episode 390: reward = -13.88, steps = 1000
16:48:59 [INFO] train episode 391: reward = -50.66, steps = 1000
16:49:17 [INFO] train episode 392: reward = 20.18, steps = 1000
16:49:33 [INFO] train episode 393: reward = -36.30, steps = 1000
16:49:51 [INFO] train episode 394: reward = -6.69, steps = 1000
16:50:08 [INFO] train episode 395: reward = -41.34, steps = 1000
16:50:24 [INFO] train episode 396: reward = 10.24, steps = 1000
16:50:41 [INFO] train episode 397: reward = -3.72, steps = 1000
16:50:59 [INFO] train episode 398: reward = 9.63, steps = 1000
16:51:15 [INFO] train episode 399: reward = 3.34, steps = 1000
16:51:33 [INFO] train episode 400: reward = -26.98, steps = 1000
16:51:50 [INFO] train episode 401: reward = -10.55, steps = 1000
16:52:08 [INFO] train episode 402: reward = -61.39, steps = 1000
16:52:25 [INFO] train episode 403: reward = -3.59, steps = 1000
16:52:41 [INFO] train episode 404: reward = -11.97, steps = 1000
16:52:59 [INFO] train episode 405: reward = 18.61, steps = 1000
16:53:15 [INFO] train episode 406: reward = -3.39, steps = 1000
16:53:34 [INFO] train episode 407: reward = -15.20, steps = 1000
16:53:51 [INFO] train episode 408: reward = -45.63, steps = 1000
16:54:07 [INFO] train episode 409: reward = -18.62, steps = 1000
16:54:24 [INFO] train episode 410: reward = 5.25, steps = 1000
16:54:41 [INFO] train episode 411: reward = -11.82, steps = 1000
16:54:58 [INFO] train episode 412: reward = 8.86, steps = 1000
16:55:15 [INFO] train episode 413: reward = 1.93, steps = 1000
16:55:33 [INFO] train episode 414: reward = -2.19, steps = 1000
16:55:50 [INFO] train episode 415: reward = -9.91, steps = 1000
16:56:08 [INFO] train episode 416: reward = -13.03, steps = 1000
16:56:25 [INFO] train episode 417: reward = -60.89, steps = 1000
16:56:42 [INFO] train episode 418: reward = -30.01, steps = 1000
16:57:00 [INFO] train episode 419: reward = -22.01, steps = 1000
16:57:17 [INFO] train episode 420: reward = -30.74, steps = 1000
16:57:35 [INFO] train episode 421: reward = 25.21, steps = 1000
16:57:52 [INFO] train episode 422: reward = -28.84, steps = 1000
16:58:09 [INFO] train episode 423: reward = -16.14, steps = 1000
16:58:26 [INFO] train episode 424: reward = -12.96, steps = 1000
16:58:43 [INFO] train episode 425: reward = -6.68, steps = 1000
16:59:00 [INFO] train episode 426: reward = -6.48, steps = 1000
16:59:18 [INFO] train episode 427: reward = 20.64, steps = 1000
16:59:35 [INFO] train episode 428: reward = -36.57, steps = 1000
16:59:46 [INFO] train episode 429: reward = 179.74, steps = 641
16:59:59 [INFO] train episode 430: reward = -258.31, steps = 750
17:00:16 [INFO] train episode 431: reward = -37.56, steps = 1000
17:00:33 [INFO] train episode 432: reward = -9.76, steps = 1000
17:00:45 [INFO] train episode 433: reward = -380.50, steps = 759
17:01:02 [INFO] train episode 434: reward = -113.16, steps = 1000
17:01:11 [INFO] train episode 435: reward = 192.68, steps = 634
17:01:28 [INFO] train episode 436: reward = -16.30, steps = 1000
17:01:37 [INFO] train episode 437: reward = 225.65, steps = 604
17:01:48 [INFO] train episode 438: reward = -92.96, steps = 670
17:02:06 [INFO] train episode 439: reward = 0.91, steps = 1000
17:02:14 [INFO] train episode 440: reward = -199.07, steps = 533
17:02:27 [INFO] train episode 441: reward = 148.87, steps = 734
17:02:41 [INFO] train episode 442: reward = 123.26, steps = 890
17:02:58 [INFO] train episode 443: reward = -81.29, steps = 1000
17:03:04 [INFO] train episode 444: reward = 197.81, steps = 401
17:03:21 [INFO] train episode 445: reward = 12.10, steps = 1000
17:03:32 [INFO] train episode 446: reward = 171.38, steps = 632
17:03:48 [INFO] train episode 447: reward = 0.39, steps = 1000
17:03:56 [INFO] train episode 448: reward = -50.86, steps = 515
17:04:12 [INFO] train episode 449: reward = -50.66, steps = 1000
17:04:25 [INFO] train episode 450: reward = -119.16, steps = 793
17:04:36 [INFO] train episode 451: reward = -162.38, steps = 723
17:04:47 [INFO] train episode 452: reward = 106.41, steps = 674
17:05:04 [INFO] train episode 453: reward = 18.17, steps = 1000
17:05:21 [INFO] train episode 454: reward = -76.30, steps = 1000
17:05:37 [INFO] train episode 455: reward = -228.40, steps = 963
17:05:54 [INFO] train episode 456: reward = -4.35, steps = 1000
17:06:11 [INFO] train episode 457: reward = 118.99, steps = 997
17:06:23 [INFO] train episode 458: reward = 171.42, steps = 770
17:06:36 [INFO] train episode 459: reward = 170.09, steps = 784
17:06:54 [INFO] train episode 460: reward = -48.00, steps = 1000
17:07:01 [INFO] train episode 461: reward = 161.24, steps = 483
17:07:08 [INFO] train episode 462: reward = -1011.54, steps = 434
17:07:24 [INFO] train episode 463: reward = 27.88, steps = 1000
17:07:40 [INFO] train episode 464: reward = -55.39, steps = 1000
17:07:57 [INFO] train episode 465: reward = -11.84, steps = 1000
17:08:13 [INFO] train episode 466: reward = -16.58, steps = 1000
17:08:31 [INFO] train episode 467: reward = -48.79, steps = 1000
17:08:44 [INFO] train episode 468: reward = 75.98, steps = 803
17:09:01 [INFO] train episode 469: reward = 73.27, steps = 984
17:09:18 [INFO] train episode 470: reward = 17.57, steps = 1000
17:09:35 [INFO] train episode 471: reward = -40.56, steps = 1000
17:09:46 [INFO] train episode 472: reward = 153.79, steps = 692
17:10:03 [INFO] train episode 473: reward = -31.45, steps = 1000
17:10:21 [INFO] train episode 474: reward = -11.71, steps = 1000
17:10:31 [INFO] train episode 475: reward = 173.09, steps = 627
17:10:47 [INFO] train episode 476: reward = -41.60, steps = 1000
17:10:56 [INFO] train episode 477: reward = 189.14, steps = 554
17:11:12 [INFO] train episode 478: reward = -28.12, steps = 1000
17:11:29 [INFO] train episode 479: reward = 33.71, steps = 1000
17:11:37 [INFO] train episode 480: reward = 248.13, steps = 464
17:11:47 [INFO] train episode 481: reward = -178.47, steps = 601
17:12:04 [INFO] train episode 482: reward = 34.76, steps = 1000
17:12:10 [INFO] train episode 483: reward = 222.54, steps = 386
17:12:27 [INFO] train episode 484: reward = -5.07, steps = 1000
17:12:34 [INFO] train episode 485: reward = 237.54, steps = 418
17:12:51 [INFO] train episode 486: reward = -422.23, steps = 986
17:13:09 [INFO] train episode 487: reward = 0.66, steps = 1000
17:13:25 [INFO] train episode 488: reward = 22.06, steps = 1000
17:13:42 [INFO] train episode 489: reward = 4.54, steps = 1000
17:13:59 [INFO] train episode 490: reward = -49.61, steps = 1000
17:14:16 [INFO] train episode 491: reward = -24.64, steps = 1000
17:14:23 [INFO] train episode 492: reward = 227.28, steps = 411
17:14:40 [INFO] train episode 493: reward = -62.41, steps = 1000
17:14:53 [INFO] train episode 494: reward = 140.25, steps = 790
17:15:10 [INFO] train episode 495: reward = 7.08, steps = 1000
17:15:28 [INFO] train episode 496: reward = 37.71, steps = 1000
17:15:45 [INFO] train episode 497: reward = 2.57, steps = 1000
17:15:51 [INFO] train episode 498: reward = 238.87, steps = 400
17:16:08 [INFO] train episode 499: reward = -4.74, steps = 1000
17:16:17 [INFO] train episode 500: reward = 215.58, steps = 544
17:16:34 [INFO] train episode 501: reward = 26.46, steps = 1000
17:16:51 [INFO] train episode 502: reward = 20.71, steps = 1000
17:17:09 [INFO] train episode 503: reward = 23.39, steps = 1000
17:17:26 [INFO] train episode 504: reward = 10.38, steps = 1000
17:17:35 [INFO] train episode 505: reward = 212.92, steps = 620
17:17:42 [INFO] train episode 506: reward = 214.52, steps = 421
17:17:47 [INFO] train episode 507: reward = -604.26, steps = 304
17:17:51 [INFO] train episode 508: reward = 220.46, steps = 304
17:18:07 [INFO] train episode 509: reward = 19.86, steps = 1000
17:18:24 [INFO] train episode 510: reward = 69.63, steps = 1000
17:18:42 [INFO] train episode 511: reward = 39.31, steps = 1000
17:18:53 [INFO] train episode 512: reward = 177.56, steps = 684
17:18:55 [INFO] train episode 513: reward = -302.84, steps = 152
17:19:12 [INFO] train episode 514: reward = 18.61, steps = 1000
17:19:14 [INFO] train episode 515: reward = -466.55, steps = 125
17:19:20 [INFO] train episode 516: reward = -415.81, steps = 408
17:19:23 [INFO] train episode 517: reward = -340.35, steps = 184
17:19:38 [INFO] train episode 518: reward = -458.65, steps = 926
17:19:42 [INFO] train episode 519: reward = 281.32, steps = 289
17:19:59 [INFO] train episode 520: reward = 5.11, steps = 1000
17:20:17 [INFO] train episode 521: reward = 39.16, steps = 1000
17:20:33 [INFO] train episode 522: reward = -30.10, steps = 1000
17:20:50 [INFO] train episode 523: reward = 21.39, steps = 1000
17:21:07 [INFO] train episode 524: reward = 26.32, steps = 1000
17:21:09 [INFO] train episode 525: reward = -57.20, steps = 135
17:21:16 [INFO] train episode 526: reward = -432.65, steps = 437
17:21:18 [INFO] train episode 527: reward = -295.68, steps = 102
17:21:19 [INFO] train episode 528: reward = -414.96, steps = 108
17:21:24 [INFO] train episode 529: reward = 236.73, steps = 304
17:21:41 [INFO] train episode 530: reward = -12.81, steps = 1000
17:21:44 [INFO] train episode 531: reward = -136.83, steps = 200
17:22:01 [INFO] train episode 532: reward = 126.82, steps = 932
17:22:19 [INFO] train episode 533: reward = -967.64, steps = 999
17:22:36 [INFO] train episode 534: reward = 64.86, steps = 1000
17:22:53 [INFO] train episode 535: reward = 85.45, steps = 1000
17:23:09 [INFO] train episode 536: reward = -13.24, steps = 1000
17:23:27 [INFO] train episode 537: reward = 41.51, steps = 1000
17:23:44 [INFO] train episode 538: reward = 35.68, steps = 1000
17:24:01 [INFO] train episode 539: reward = -11.86, steps = 1000
17:24:09 [INFO] train episode 540: reward = -549.14, steps = 509
17:24:10 [INFO] train episode 541: reward = -53.61, steps = 74
17:24:28 [INFO] train episode 542: reward = -25.94, steps = 1000
17:24:45 [INFO] train episode 543: reward = -13.76, steps = 1000
17:24:52 [INFO] train episode 544: reward = -610.65, steps = 391
17:25:07 [INFO] train episode 545: reward = -133.54, steps = 905
17:25:25 [INFO] train episode 546: reward = -2.68, steps = 1000
17:25:42 [INFO] train episode 547: reward = 32.10, steps = 1000
17:25:43 [INFO] train episode 548: reward = 22.49, steps = 96
17:25:45 [INFO] train episode 549: reward = 33.30, steps = 102
17:26:02 [INFO] train episode 550: reward = -39.32, steps = 1000
17:26:19 [INFO] train episode 551: reward = -37.75, steps = 1000
17:26:38 [INFO] train episode 552: reward = 30.53, steps = 1000
17:26:55 [INFO] train episode 553: reward = -15.91, steps = 1000
17:27:12 [INFO] train episode 554: reward = -4.25, steps = 1000
17:27:29 [INFO] train episode 555: reward = -6.59, steps = 1000
17:27:46 [INFO] train episode 556: reward = 28.12, steps = 1000
17:28:03 [INFO] train episode 557: reward = 41.68, steps = 1000
17:28:20 [INFO] train episode 558: reward = 71.26, steps = 1000
17:28:37 [INFO] train episode 559: reward = 22.88, steps = 1000
17:28:52 [INFO] train episode 560: reward = 195.52, steps = 892
17:29:09 [INFO] train episode 561: reward = 24.32, steps = 1000
17:29:26 [INFO] train episode 562: reward = -19.14, steps = 1000
17:29:44 [INFO] train episode 563: reward = 56.24, steps = 1000
17:30:01 [INFO] train episode 564: reward = 85.08, steps = 1000
17:30:15 [INFO] train episode 565: reward = 174.13, steps = 824
17:30:17 [INFO] train episode 566: reward = -178.52, steps = 168
17:30:35 [INFO] train episode 567: reward = -2.73, steps = 1000
17:30:52 [INFO] train episode 568: reward = 37.31, steps = 1000
17:30:59 [INFO] train episode 569: reward = 215.90, steps = 448
17:31:16 [INFO] train episode 570: reward = -5.80, steps = 1000
17:31:17 [INFO] train episode 571: reward = -51.08, steps = 84
17:31:34 [INFO] train episode 572: reward = 37.96, steps = 1000
17:31:52 [INFO] train episode 573: reward = 38.76, steps = 1000
17:32:08 [INFO] train episode 574: reward = 12.21, steps = 1000
17:32:10 [INFO] train episode 575: reward = -82.36, steps = 96
17:32:16 [INFO] train episode 576: reward = 217.38, steps = 405
17:32:25 [INFO] train episode 577: reward = 203.25, steps = 565
17:32:36 [INFO] train episode 578: reward = 221.50, steps = 640
17:32:38 [INFO] train episode 579: reward = 76.11, steps = 159
17:32:55 [INFO] train episode 580: reward = 132.48, steps = 977
17:33:05 [INFO] train episode 581: reward = 219.43, steps = 614
17:33:21 [INFO] train episode 582: reward = -9.71, steps = 1000
17:33:27 [INFO] train episode 583: reward = 240.53, steps = 365
17:33:44 [INFO] train episode 584: reward = 2.31, steps = 1000
17:33:45 [INFO] train episode 585: reward = 32.22, steps = 89
17:33:47 [INFO] train episode 586: reward = -27.33, steps = 86
17:33:51 [INFO] train episode 587: reward = 258.94, steps = 308
17:34:00 [INFO] train episode 588: reward = 241.40, steps = 566
17:34:18 [INFO] train episode 589: reward = 59.84, steps = 1000
17:34:20 [INFO] train episode 590: reward = -59.60, steps = 108
17:34:21 [INFO] train episode 591: reward = -34.47, steps = 57
17:34:37 [INFO] train episode 592: reward = 31.24, steps = 1000
17:34:51 [INFO] train episode 593: reward = 187.36, steps = 800
17:35:03 [INFO] train episode 594: reward = 197.85, steps = 713
17:35:21 [INFO] train episode 595: reward = 10.25, steps = 1000
17:35:38 [INFO] train episode 596: reward = 17.86, steps = 1000
17:35:55 [INFO] train episode 597: reward = 41.72, steps = 1000
17:35:57 [INFO] train episode 598: reward = 2.79, steps = 121
17:36:14 [INFO] train episode 599: reward = 53.77, steps = 1000
17:36:31 [INFO] train episode 600: reward = -1.95, steps = 1000
17:36:41 [INFO] train episode 601: reward = 207.04, steps = 608
17:36:58 [INFO] train episode 602: reward = 42.99, steps = 1000
17:37:16 [INFO] train episode 603: reward = 25.57, steps = 1000
17:37:24 [INFO] train episode 604: reward = 202.82, steps = 509
17:37:40 [INFO] train episode 605: reward = 45.00, steps = 1000
17:37:58 [INFO] train episode 606: reward = 38.45, steps = 1000
17:38:15 [INFO] train episode 607: reward = 41.12, steps = 1000
17:38:16 [INFO] train episode 608: reward = -14.02, steps = 106
17:38:33 [INFO] train episode 609: reward = 20.66, steps = 1000
17:38:51 [INFO] train episode 610: reward = 21.48, steps = 1000
17:39:04 [INFO] train episode 611: reward = 175.96, steps = 788
17:39:21 [INFO] train episode 612: reward = 51.15, steps = 1000
17:39:38 [INFO] train episode 613: reward = 1.04, steps = 1000
17:39:55 [INFO] train episode 614: reward = 114.43, steps = 1000
17:40:00 [INFO] train episode 615: reward = 238.75, steps = 321
17:40:17 [INFO] train episode 616: reward = 105.67, steps = 1000
17:40:33 [INFO] train episode 617: reward = 109.79, steps = 1000
17:40:49 [INFO] train episode 618: reward = 176.95, steps = 990
17:41:06 [INFO] train episode 619: reward = 54.18, steps = 1000
17:41:17 [INFO] train episode 620: reward = 214.27, steps = 640
17:41:23 [INFO] train episode 621: reward = 204.66, steps = 389
17:41:25 [INFO] train episode 622: reward = 7.88, steps = 119
17:41:41 [INFO] train episode 623: reward = 165.24, steps = 1000
17:41:57 [INFO] train episode 624: reward = 167.14, steps = 1000
17:41:58 [INFO] train episode 625: reward = 7.88, steps = 72
17:42:00 [INFO] train episode 626: reward = -128.61, steps = 91
17:42:01 [INFO] train episode 627: reward = -16.84, steps = 74
17:42:17 [INFO] train episode 628: reward = 155.34, steps = 1000
17:42:33 [INFO] train episode 629: reward = 100.81, steps = 1000
17:42:35 [INFO] train episode 630: reward = -115.57, steps = 117
17:42:51 [INFO] train episode 631: reward = 138.67, steps = 1000
17:43:09 [INFO] train episode 632: reward = 14.39, steps = 1000
17:43:26 [INFO] train episode 633: reward = 102.26, steps = 1000
17:43:40 [INFO] train episode 634: reward = 181.22, steps = 861
17:43:56 [INFO] train episode 635: reward = 164.01, steps = 1000
17:44:01 [INFO] train episode 636: reward = 235.82, steps = 343
17:44:17 [INFO] train episode 637: reward = 162.08, steps = 1000
17:44:19 [INFO] train episode 638: reward = 10.27, steps = 128
17:44:36 [INFO] train episode 639: reward = 117.35, steps = 1000
17:44:38 [INFO] train episode 640: reward = 53.56, steps = 111
17:44:47 [INFO] train episode 641: reward = 256.07, steps = 595
17:45:06 [INFO] train episode 642: reward = 101.01, steps = 1000
17:45:23 [INFO] train episode 643: reward = 122.99, steps = 1000
17:45:25 [INFO] train episode 644: reward = 73.47, steps = 120
17:45:42 [INFO] train episode 645: reward = 142.42, steps = 1000
17:45:44 [INFO] train episode 646: reward = 66.60, steps = 104
17:45:53 [INFO] train episode 647: reward = 220.20, steps = 564
17:46:10 [INFO] train episode 648: reward = 127.84, steps = 1000
17:46:26 [INFO] train episode 649: reward = 120.23, steps = 1000
17:46:33 [INFO] train episode 650: reward = 223.96, steps = 461
17:46:49 [INFO] train episode 651: reward = 154.65, steps = 1000
17:47:06 [INFO] train episode 652: reward = 143.88, steps = 1000
17:47:22 [INFO] train episode 653: reward = 150.57, steps = 1000
17:47:38 [INFO] train episode 654: reward = 27.69, steps = 1000
17:47:42 [INFO] train episode 655: reward = -43.86, steps = 229
17:47:57 [INFO] train episode 656: reward = 89.57, steps = 1000
17:48:14 [INFO] train episode 657: reward = 168.64, steps = 1000
17:48:24 [INFO] train episode 658: reward = 207.00, steps = 642
17:48:40 [INFO] train episode 659: reward = 182.23, steps = 1000
17:48:57 [INFO] train episode 660: reward = 115.71, steps = 1000
17:49:01 [INFO] train episode 661: reward = -160.52, steps = 281
17:49:17 [INFO] train episode 662: reward = 157.85, steps = 1000
17:49:33 [INFO] train episode 663: reward = 130.31, steps = 1000
17:49:49 [INFO] train episode 664: reward = 130.32, steps = 1000
17:49:56 [INFO] train episode 665: reward = 277.95, steps = 410
17:50:12 [INFO] train episode 666: reward = 154.16, steps = 1000
17:50:18 [INFO] train episode 667: reward = 196.92, steps = 423
17:50:34 [INFO] train episode 668: reward = 135.52, steps = 1000
17:50:50 [INFO] train episode 669: reward = 106.39, steps = 1000
17:50:52 [INFO] train episode 670: reward = -24.90, steps = 104
17:51:08 [INFO] train episode 671: reward = 154.71, steps = 1000
17:51:15 [INFO] train episode 672: reward = 226.01, steps = 468
17:51:31 [INFO] train episode 673: reward = 132.32, steps = 1000
17:51:47 [INFO] train episode 674: reward = 138.69, steps = 1000
17:52:00 [INFO] train episode 675: reward = 232.37, steps = 797
17:52:13 [INFO] train episode 676: reward = 249.46, steps = 817
17:52:29 [INFO] train episode 677: reward = 112.30, steps = 1000
17:52:45 [INFO] train episode 678: reward = 104.20, steps = 1000
17:52:51 [INFO] train episode 679: reward = -187.01, steps = 379
17:53:08 [INFO] train episode 680: reward = 136.29, steps = 1000
17:53:13 [INFO] train episode 681: reward = -486.16, steps = 360
17:53:20 [INFO] train episode 682: reward = -749.46, steps = 446
17:53:36 [INFO] train episode 683: reward = 109.68, steps = 1000
17:53:52 [INFO] train episode 684: reward = 154.85, steps = 1000
17:54:09 [INFO] train episode 685: reward = 156.93, steps = 1000
17:54:12 [INFO] train episode 686: reward = -406.34, steps = 210
17:54:17 [INFO] train episode 687: reward = -401.19, steps = 300
17:54:21 [INFO] train episode 688: reward = -82.41, steps = 291
17:54:37 [INFO] train episode 689: reward = 115.55, steps = 1000
17:54:54 [INFO] train episode 690: reward = 121.94, steps = 1000
17:55:02 [INFO] train episode 691: reward = 183.99, steps = 519
17:55:19 [INFO] train episode 692: reward = 76.99, steps = 1000
17:55:37 [INFO] train episode 693: reward = 57.91, steps = 1000
17:55:53 [INFO] train episode 694: reward = 123.04, steps = 1000
17:56:10 [INFO] train episode 695: reward = 38.86, steps = 1000
17:56:25 [INFO] train episode 696: reward = 134.54, steps = 1000
17:56:32 [INFO] train episode 697: reward = -130.50, steps = 430
17:56:49 [INFO] train episode 698: reward = 124.60, steps = 1000
17:56:53 [INFO] train episode 699: reward = 267.28, steps = 276
17:56:59 [INFO] train episode 700: reward = 209.36, steps = 389
17:57:15 [INFO] train episode 701: reward = 159.69, steps = 1000
17:57:18 [INFO] train episode 702: reward = -7.29, steps = 189
17:57:25 [INFO] train episode 703: reward = 222.08, steps = 413
17:57:36 [INFO] train episode 704: reward = 232.52, steps = 686
17:57:40 [INFO] train episode 705: reward = 10.55, steps = 215
17:57:57 [INFO] train episode 706: reward = 142.65, steps = 1000
17:58:14 [INFO] train episode 707: reward = 171.00, steps = 1000
17:58:32 [INFO] train episode 708: reward = 128.44, steps = 1000
17:58:36 [INFO] train episode 709: reward = 4.20, steps = 241
17:58:56 [INFO] train episode 710: reward = -51.23, steps = 1000
17:59:02 [INFO] train episode 711: reward = 244.43, steps = 382
17:59:09 [INFO] train episode 712: reward = -198.80, steps = 386
17:59:26 [INFO] train episode 713: reward = 120.89, steps = 1000
17:59:43 [INFO] train episode 714: reward = 89.99, steps = 1000
17:59:47 [INFO] train episode 715: reward = 217.96, steps = 308
18:00:00 [INFO] train episode 716: reward = 127.40, steps = 741
18:00:08 [INFO] train episode 717: reward = 256.36, steps = 516
18:00:22 [INFO] train episode 718: reward = 177.99, steps = 759
18:00:33 [INFO] train episode 719: reward = 247.88, steps = 732
18:00:51 [INFO] train episode 720: reward = 130.01, steps = 1000
18:00:54 [INFO] train episode 721: reward = -313.62, steps = 177
18:01:04 [INFO] train episode 722: reward = 250.17, steps = 519
18:01:24 [INFO] train episode 723: reward = 80.82, steps = 1000
18:01:28 [INFO] train episode 724: reward = 40.78, steps = 188
18:01:48 [INFO] train episode 725: reward = 99.39, steps = 1000
18:02:06 [INFO] train episode 726: reward = 143.09, steps = 1000
18:02:25 [INFO] train episode 727: reward = 153.73, steps = 1000
18:02:42 [INFO] train episode 728: reward = 147.26, steps = 1000
18:02:52 [INFO] train episode 729: reward = 238.17, steps = 618
18:03:09 [INFO] train episode 730: reward = 79.20, steps = 1000
18:03:27 [INFO] train episode 731: reward = 24.03, steps = 1000
18:03:31 [INFO] train episode 732: reward = 242.16, steps = 235
18:03:47 [INFO] train episode 733: reward = 78.30, steps = 1000
18:04:03 [INFO] train episode 734: reward = 97.28, steps = 1000
18:04:20 [INFO] train episode 735: reward = 90.98, steps = 1000
18:04:38 [INFO] train episode 736: reward = 79.91, steps = 1000
18:04:50 [INFO] train episode 737: reward = -159.03, steps = 692
18:05:08 [INFO] train episode 738: reward = 85.60, steps = 1000
18:05:10 [INFO] train episode 739: reward = 43.50, steps = 171
18:05:13 [INFO] train episode 740: reward = 30.27, steps = 171
18:05:16 [INFO] train episode 741: reward = -454.75, steps = 188
18:05:17 [INFO] train episode 742: reward = -212.17, steps = 70
18:05:33 [INFO] train episode 743: reward = 219.78, steps = 982
18:05:50 [INFO] train episode 744: reward = 149.90, steps = 1000
18:06:07 [INFO] train episode 745: reward = 128.33, steps = 1000
18:06:15 [INFO] train episode 746: reward = 208.50, steps = 468
18:06:20 [INFO] train episode 747: reward = -88.29, steps = 306
18:06:23 [INFO] train episode 748: reward = 255.11, steps = 195
18:06:36 [INFO] train episode 749: reward = 231.50, steps = 765
18:06:38 [INFO] train episode 750: reward = 33.01, steps = 145
18:06:56 [INFO] train episode 751: reward = -55.89, steps = 1000
18:07:01 [INFO] train episode 752: reward = -7.38, steps = 328
18:07:06 [INFO] train episode 753: reward = -133.72, steps = 330
18:07:22 [INFO] train episode 754: reward = 251.94, steps = 940
18:07:38 [INFO] train episode 755: reward = 138.53, steps = 1000
18:07:40 [INFO] train episode 756: reward = 11.25, steps = 119
18:07:56 [INFO] train episode 757: reward = 151.37, steps = 1000
18:08:04 [INFO] train episode 758: reward = 241.25, steps = 475
18:08:21 [INFO] train episode 759: reward = -39.54, steps = 1000
18:08:37 [INFO] train episode 760: reward = 144.65, steps = 1000
18:08:49 [INFO] train episode 761: reward = 211.23, steps = 696
18:09:06 [INFO] train episode 762: reward = 108.09, steps = 1000
18:09:23 [INFO] train episode 763: reward = -86.33, steps = 1000
18:09:34 [INFO] train episode 764: reward = -99.00, steps = 704
18:09:45 [INFO] train episode 765: reward = 184.86, steps = 652
18:09:53 [INFO] train episode 766: reward = 256.11, steps = 444
18:09:56 [INFO] train episode 767: reward = 263.82, steps = 220
18:10:13 [INFO] train episode 768: reward = 158.73, steps = 1000
18:10:30 [INFO] train episode 769: reward = 79.91, steps = 1000
18:10:48 [INFO] train episode 770: reward = 222.30, steps = 982
18:10:52 [INFO] train episode 771: reward = 294.06, steps = 263
18:11:11 [INFO] train episode 772: reward = 130.29, steps = 1000
18:11:14 [INFO] train episode 773: reward = -66.40, steps = 179
18:11:30 [INFO] train episode 774: reward = 102.93, steps = 1000
18:11:48 [INFO] train episode 775: reward = 115.97, steps = 1000
18:12:05 [INFO] train episode 776: reward = 107.86, steps = 1000
18:12:19 [INFO] train episode 777: reward = 254.59, steps = 793
18:12:35 [INFO] train episode 778: reward = 104.27, steps = 1000
18:12:52 [INFO] train episode 779: reward = 158.35, steps = 1000
18:12:59 [INFO] train episode 780: reward = 227.04, steps = 399
18:13:16 [INFO] train episode 781: reward = 171.72, steps = 1000
18:13:21 [INFO] train episode 782: reward = 252.99, steps = 329
18:13:36 [INFO] train episode 783: reward = 261.01, steps = 894
18:13:52 [INFO] train episode 784: reward = 255.66, steps = 925
18:14:09 [INFO] train episode 785: reward = 133.93, steps = 1000
18:14:26 [INFO] train episode 786: reward = 155.62, steps = 1000
18:14:43 [INFO] train episode 787: reward = 141.15, steps = 1000
18:14:49 [INFO] train episode 788: reward = 234.30, steps = 377
18:14:54 [INFO] train episode 789: reward = 253.35, steps = 258
18:15:12 [INFO] train episode 790: reward = 84.78, steps = 1000
18:15:30 [INFO] train episode 791: reward = 117.51, steps = 1000
18:15:47 [INFO] train episode 792: reward = 93.79, steps = 1000
18:15:49 [INFO] train episode 793: reward = -185.16, steps = 129
18:16:05 [INFO] train episode 794: reward = 162.98, steps = 1000
18:16:09 [INFO] train episode 795: reward = 256.94, steps = 258
18:16:25 [INFO] train episode 796: reward = 134.97, steps = 1000
18:16:26 [INFO] train episode 797: reward = -231.05, steps = 79
18:16:43 [INFO] train episode 798: reward = 151.98, steps = 1000
18:16:59 [INFO] train episode 799: reward = 130.34, steps = 1000
18:17:15 [INFO] train episode 800: reward = 154.68, steps = 1000
18:17:19 [INFO] train episode 801: reward = -170.08, steps = 214
18:17:35 [INFO] train episode 802: reward = 81.76, steps = 1000
18:17:51 [INFO] train episode 803: reward = 140.37, steps = 1000
18:17:58 [INFO] train episode 804: reward = 215.17, steps = 409
18:18:01 [INFO] train episode 805: reward = 278.40, steps = 211
18:18:18 [INFO] train episode 806: reward = 93.31, steps = 1000
18:18:34 [INFO] train episode 807: reward = 166.68, steps = 1000
18:18:50 [INFO] train episode 808: reward = 132.64, steps = 1000
18:18:54 [INFO] train episode 809: reward = 240.51, steps = 272
18:18:58 [INFO] train episode 810: reward = 251.70, steps = 217
18:19:14 [INFO] train episode 811: reward = 127.53, steps = 1000
18:19:31 [INFO] train episode 812: reward = 95.43, steps = 1000
18:19:48 [INFO] train episode 813: reward = 135.73, steps = 1000
18:19:58 [INFO] train episode 814: reward = 268.12, steps = 661
18:20:15 [INFO] train episode 815: reward = 147.78, steps = 1000
18:20:31 [INFO] train episode 816: reward = 170.47, steps = 1000
18:20:48 [INFO] train episode 817: reward = 130.99, steps = 1000
18:20:52 [INFO] train episode 818: reward = 262.83, steps = 274
18:21:09 [INFO] train episode 819: reward = 110.39, steps = 1000
18:21:15 [INFO] train episode 820: reward = 239.54, steps = 378
18:21:31 [INFO] train episode 821: reward = 133.82, steps = 1000
18:21:48 [INFO] train episode 822: reward = 137.69, steps = 1000
18:22:05 [INFO] train episode 823: reward = 149.98, steps = 1000
18:22:21 [INFO] train episode 824: reward = 174.66, steps = 1000
18:22:23 [INFO] train episode 825: reward = 23.42, steps = 111
18:22:29 [INFO] train episode 826: reward = 256.38, steps = 376
18:22:43 [INFO] train episode 827: reward = 170.66, steps = 1000
18:22:58 [INFO] train episode 828: reward = 144.07, steps = 1000
18:23:12 [INFO] train episode 829: reward = 141.22, steps = 1000
18:23:15 [INFO] train episode 830: reward = 268.09, steps = 214
18:23:21 [INFO] train episode 831: reward = 197.00, steps = 489
18:23:28 [INFO] train episode 832: reward = 248.00, steps = 492
18:23:42 [INFO] train episode 833: reward = 125.99, steps = 1000
18:23:56 [INFO] train episode 834: reward = 136.15, steps = 1000
18:23:59 [INFO] train episode 835: reward = 269.43, steps = 251
18:24:14 [INFO] train episode 836: reward = 125.02, steps = 1000
18:24:28 [INFO] train episode 837: reward = 110.55, steps = 1000
18:24:41 [INFO] train episode 838: reward = 129.61, steps = 1000
18:24:44 [INFO] train episode 839: reward = 271.42, steps = 227
18:24:46 [INFO] train episode 840: reward = 19.81, steps = 139
18:25:00 [INFO] train episode 841: reward = 131.76, steps = 1000
18:25:06 [INFO] train episode 842: reward = 294.54, steps = 406
18:25:20 [INFO] train episode 843: reward = 104.78, steps = 1000
18:25:34 [INFO] train episode 844: reward = 99.10, steps = 1000
18:25:48 [INFO] train episode 845: reward = 115.60, steps = 1000
18:26:02 [INFO] train episode 846: reward = 140.14, steps = 1000
18:26:17 [INFO] train episode 847: reward = 145.72, steps = 1000
18:26:31 [INFO] train episode 848: reward = 165.32, steps = 1000
18:26:46 [INFO] train episode 849: reward = 132.30, steps = 1000
18:27:01 [INFO] train episode 850: reward = 139.42, steps = 1000
18:27:05 [INFO] train episode 851: reward = -866.85, steps = 260
18:27:11 [INFO] train episode 852: reward = -1546.72, steps = 403
18:27:16 [INFO] train episode 853: reward = -161.36, steps = 358
18:27:21 [INFO] train episode 854: reward = 212.41, steps = 378
18:27:22 [INFO] train episode 855: reward = -604.72, steps = 92
18:27:30 [INFO] train episode 856: reward = 255.88, steps = 556
18:27:32 [INFO] train episode 857: reward = 4.19, steps = 188
18:27:34 [INFO] train episode 858: reward = 261.39, steps = 156
18:27:36 [INFO] train episode 859: reward = -7.95, steps = 86
18:27:44 [INFO] train episode 860: reward = 218.55, steps = 613
18:27:47 [INFO] train episode 861: reward = 289.51, steps = 241
18:27:50 [INFO] train episode 862: reward = 246.36, steps = 255
18:27:51 [INFO] train episode 863: reward = 26.10, steps = 75
18:28:05 [INFO] train episode 864: reward = 157.62, steps = 1000
18:28:19 [INFO] train episode 865: reward = 115.23, steps = 1000
18:28:22 [INFO] train episode 866: reward = -328.90, steps = 274
18:28:26 [INFO] train episode 867: reward = 230.80, steps = 280
18:28:32 [INFO] train episode 868: reward = 259.59, steps = 456
18:28:36 [INFO] train episode 869: reward = -585.37, steps = 340
18:28:38 [INFO] train episode 870: reward = 28.53, steps = 149
18:28:41 [INFO] train episode 871: reward = 284.25, steps = 244
18:28:55 [INFO] train episode 872: reward = 137.13, steps = 1000
18:28:56 [INFO] train episode 873: reward = -52.76, steps = 75
18:29:00 [INFO] train episode 874: reward = -44.45, steps = 263
18:29:02 [INFO] train episode 875: reward = 53.58, steps = 148
18:29:16 [INFO] train episode 876: reward = 113.73, steps = 1000
18:29:18 [INFO] train episode 877: reward = -65.56, steps = 218
18:29:19 [INFO] train episode 878: reward = -87.05, steps = 68
18:29:24 [INFO] train episode 879: reward = 253.74, steps = 325
18:29:30 [INFO] train episode 880: reward = -179.31, steps = 483
18:29:33 [INFO] train episode 881: reward = 249.55, steps = 205
18:29:35 [INFO] train episode 882: reward = -5.75, steps = 202
18:29:40 [INFO] train episode 883: reward = 238.89, steps = 340
18:29:46 [INFO] train episode 884: reward = 228.81, steps = 456
18:29:47 [INFO] train episode 885: reward = 13.06, steps = 115
18:29:56 [INFO] train episode 886: reward = 243.02, steps = 598
18:30:01 [INFO] train episode 887: reward = 209.74, steps = 400
18:30:04 [INFO] train episode 888: reward = 279.38, steps = 234
18:30:18 [INFO] train episode 889: reward = 128.43, steps = 1000
18:30:33 [INFO] train episode 890: reward = 85.24, steps = 1000
18:30:48 [INFO] train episode 891: reward = 111.26, steps = 1000
18:31:03 [INFO] train episode 892: reward = 157.65, steps = 1000
18:31:11 [INFO] train episode 893: reward = -24.41, steps = 577
18:31:19 [INFO] train episode 894: reward = 218.77, steps = 587
18:31:23 [INFO] train episode 895: reward = 218.10, steps = 283
18:31:37 [INFO] train episode 896: reward = 108.42, steps = 1000
18:31:46 [INFO] train episode 897: reward = -63.21, steps = 600
18:31:51 [INFO] train episode 898: reward = 192.27, steps = 346
18:31:57 [INFO] train episode 899: reward = 248.43, steps = 459
18:32:01 [INFO] train episode 900: reward = 269.86, steps = 267
18:32:03 [INFO] train episode 901: reward = 63.01, steps = 141
18:32:09 [INFO] train episode 902: reward = 250.85, steps = 484
18:32:12 [INFO] train episode 903: reward = -231.04, steps = 230
18:32:18 [INFO] train episode 904: reward = 231.72, steps = 440
18:32:29 [INFO] train episode 905: reward = 225.02, steps = 811
18:32:35 [INFO] train episode 906: reward = 253.08, steps = 421
18:32:49 [INFO] train episode 907: reward = 130.34, steps = 1000
18:32:59 [INFO] train episode 908: reward = 213.74, steps = 704
18:33:14 [INFO] train episode 909: reward = 91.86, steps = 1000
18:33:18 [INFO] train episode 910: reward = 295.56, steps = 299
18:33:23 [INFO] train episode 911: reward = 196.36, steps = 395
18:33:38 [INFO] train episode 912: reward = 164.99, steps = 1000
18:33:52 [INFO] train episode 913: reward = 131.58, steps = 1000
18:34:06 [INFO] train episode 914: reward = 135.81, steps = 1000
18:34:20 [INFO] train episode 915: reward = 152.15, steps = 1000
18:34:34 [INFO] train episode 916: reward = 183.03, steps = 1000
18:34:48 [INFO] train episode 917: reward = 133.81, steps = 1000
18:35:02 [INFO] train episode 918: reward = 130.18, steps = 1000
18:35:09 [INFO] train episode 919: reward = 254.73, steps = 509
18:35:23 [INFO] train episode 920: reward = 109.03, steps = 1000
18:35:38 [INFO] train episode 921: reward = 166.15, steps = 1000
18:35:52 [INFO] train episode 922: reward = 142.53, steps = 1000
18:36:07 [INFO] train episode 923: reward = 142.92, steps = 1000
18:36:22 [INFO] train episode 924: reward = 167.44, steps = 1000
18:36:30 [INFO] train episode 925: reward = 188.15, steps = 505
18:36:43 [INFO] train episode 926: reward = 215.21, steps = 874
18:36:58 [INFO] train episode 927: reward = 143.08, steps = 1000
18:37:13 [INFO] train episode 928: reward = 112.91, steps = 1000
18:37:28 [INFO] train episode 929: reward = 81.78, steps = 1000
18:37:42 [INFO] train episode 930: reward = 140.03, steps = 1000
18:37:55 [INFO] train episode 931: reward = 163.42, steps = 1000
18:38:09 [INFO] train episode 932: reward = 98.82, steps = 1000
18:38:23 [INFO] train episode 933: reward = 120.41, steps = 1000
18:38:37 [INFO] train episode 934: reward = 183.47, steps = 1000
18:38:52 [INFO] train episode 935: reward = 137.67, steps = 1000
18:38:59 [INFO] train episode 936: reward = 266.40, steps = 543
18:39:03 [INFO] train episode 937: reward = 297.45, steps = 288
18:39:07 [INFO] train episode 938: reward = 260.39, steps = 280
18:39:15 [INFO] train episode 939: reward = 207.47, steps = 613
18:39:17 [INFO] train episode 940: reward = -28.38, steps = 104
18:39:22 [INFO] train episode 941: reward = 257.70, steps = 377
18:39:28 [INFO] train episode 942: reward = 254.55, steps = 472
18:39:42 [INFO] train episode 943: reward = 149.70, steps = 1000
18:39:56 [INFO] train episode 944: reward = 145.37, steps = 1000
18:40:01 [INFO] train episode 945: reward = 258.16, steps = 300
18:40:03 [INFO] train episode 946: reward = 72.58, steps = 151
18:40:11 [INFO] train episode 947: reward = 265.94, steps = 506
18:40:14 [INFO] train episode 948: reward = 287.80, steps = 271
18:40:19 [INFO] train episode 949: reward = 271.69, steps = 364
18:40:21 [INFO] train episode 950: reward = 9.70, steps = 129
18:40:35 [INFO] train episode 951: reward = 223.55, steps = 970
18:40:41 [INFO] train episode 952: reward = 242.70, steps = 404
18:40:56 [INFO] train episode 953: reward = 137.20, steps = 1000
18:41:10 [INFO] train episode 954: reward = 131.76, steps = 1000
18:41:24 [INFO] train episode 955: reward = 161.45, steps = 1000
18:41:27 [INFO] train episode 956: reward = 284.07, steps = 204
18:41:42 [INFO] train episode 957: reward = 145.33, steps = 1000
18:41:58 [INFO] train episode 958: reward = 138.87, steps = 1000
18:42:11 [INFO] train episode 959: reward = 240.89, steps = 874
18:42:16 [INFO] train episode 960: reward = 247.68, steps = 373
18:42:24 [INFO] train episode 961: reward = 275.73, steps = 607
18:42:40 [INFO] train episode 962: reward = 147.51, steps = 1000
18:42:56 [INFO] train episode 963: reward = 130.93, steps = 1000
18:43:10 [INFO] train episode 964: reward = 143.72, steps = 1000
18:43:13 [INFO] train episode 965: reward = 10.94, steps = 185
18:43:16 [INFO] train episode 966: reward = 279.68, steps = 229
18:43:30 [INFO] train episode 967: reward = 116.36, steps = 1000
18:43:44 [INFO] train episode 968: reward = 116.62, steps = 1000
18:43:59 [INFO] train episode 969: reward = 110.02, steps = 1000
18:44:02 [INFO] train episode 970: reward = -42.73, steps = 198
18:44:13 [INFO] train episode 971: reward = 270.28, steps = 778
18:44:28 [INFO] train episode 972: reward = 143.24, steps = 1000
18:44:41 [INFO] train episode 973: reward = 238.53, steps = 834
18:44:43 [INFO] train episode 974: reward = 25.61, steps = 120
18:44:58 [INFO] train episode 975: reward = 114.87, steps = 1000
18:45:00 [INFO] train episode 976: reward = 246.41, steps = 182
18:45:04 [INFO] train episode 977: reward = 272.81, steps = 274
18:45:07 [INFO] train episode 978: reward = 284.30, steps = 156
18:45:09 [INFO] train episode 979: reward = 275.49, steps = 181
18:45:11 [INFO] train episode 980: reward = -188.34, steps = 128
18:45:26 [INFO] train episode 981: reward = 148.86, steps = 1000
18:45:34 [INFO] train episode 982: reward = 213.56, steps = 496
18:45:49 [INFO] train episode 983: reward = 109.34, steps = 1000
18:45:52 [INFO] train episode 984: reward = 234.93, steps = 235
18:46:07 [INFO] train episode 985: reward = 130.47, steps = 1000
18:46:11 [INFO] train episode 986: reward = 248.10, steps = 298
18:46:13 [INFO] train episode 987: reward = 3.70, steps = 143
18:46:14 [INFO] train episode 988: reward = 20.89, steps = 137
18:46:18 [INFO] train episode 989: reward = 222.76, steps = 261
18:46:24 [INFO] train episode 990: reward = 288.13, steps = 429
18:46:28 [INFO] train episode 991: reward = 244.69, steps = 295
18:46:32 [INFO] train episode 992: reward = 262.47, steps = 337
18:46:35 [INFO] train episode 993: reward = 281.84, steps = 226
18:46:39 [INFO] train episode 994: reward = 261.27, steps = 296
18:46:53 [INFO] train episode 995: reward = 172.76, steps = 1000
18:47:08 [INFO] train episode 996: reward = 129.99, steps = 1000
18:47:22 [INFO] train episode 997: reward = 146.06, steps = 1000
18:47:36 [INFO] train episode 998: reward = 145.55, steps = 1000
18:47:50 [INFO] train episode 999: reward = 166.10, steps = 1000
18:47:53 [INFO] train episode 1000: reward = 271.57, steps = 228
18:47:55 [INFO] train episode 1001: reward = 56.52, steps = 134
18:48:01 [INFO] train episode 1002: reward = 271.93, steps = 501
18:48:05 [INFO] train episode 1003: reward = 285.02, steps = 295
18:48:20 [INFO] train episode 1004: reward = 133.74, steps = 1000
18:48:34 [INFO] train episode 1005: reward = 113.48, steps = 1000
18:48:38 [INFO] train episode 1006: reward = 222.72, steps = 323
18:48:45 [INFO] train episode 1007: reward = 279.40, steps = 535
18:49:00 [INFO] train episode 1008: reward = 128.84, steps = 1000
18:49:03 [INFO] train episode 1009: reward = 246.11, steps = 217
18:49:17 [INFO] train episode 1010: reward = 132.16, steps = 1000
18:49:20 [INFO] train episode 1011: reward = -199.85, steps = 209
18:49:23 [INFO] train episode 1012: reward = 257.34, steps = 227
18:49:28 [INFO] train episode 1013: reward = 278.25, steps = 371
18:49:32 [INFO] train episode 1014: reward = 243.59, steps = 336
18:49:34 [INFO] train episode 1015: reward = -37.41, steps = 83
18:49:41 [INFO] train episode 1016: reward = 244.25, steps = 571
18:49:56 [INFO] train episode 1017: reward = 171.17, steps = 1000
18:49:57 [INFO] train episode 1018: reward = 24.91, steps = 110
18:49:59 [INFO] train episode 1019: reward = 38.19, steps = 154
18:50:14 [INFO] train episode 1020: reward = 103.89, steps = 1000
18:50:18 [INFO] train episode 1021: reward = 279.42, steps = 298
18:50:32 [INFO] train episode 1022: reward = 171.37, steps = 1000
18:50:47 [INFO] train episode 1023: reward = 141.96, steps = 1000
18:51:01 [INFO] train episode 1024: reward = 131.08, steps = 1000
18:51:03 [INFO] train episode 1025: reward = 70.47, steps = 113
18:51:07 [INFO] train episode 1026: reward = 244.15, steps = 326
18:51:22 [INFO] train episode 1027: reward = 86.42, steps = 1000
18:51:25 [INFO] train episode 1028: reward = 225.56, steps = 233
18:51:27 [INFO] train episode 1029: reward = -229.27, steps = 117
18:51:33 [INFO] train episode 1030: reward = 248.30, steps = 471
18:51:47 [INFO] train episode 1031: reward = 141.96, steps = 1000
18:52:02 [INFO] train episode 1032: reward = 158.02, steps = 1000
18:52:16 [INFO] train episode 1033: reward = 126.78, steps = 1000
18:52:17 [INFO] train episode 1034: reward = -54.74, steps = 58
18:52:31 [INFO] train episode 1035: reward = 130.50, steps = 1000
18:52:45 [INFO] train episode 1036: reward = 134.02, steps = 1000
18:52:50 [INFO] train episode 1037: reward = 273.50, steps = 326
18:53:05 [INFO] train episode 1038: reward = 113.90, steps = 1000
18:53:10 [INFO] train episode 1039: reward = 231.13, steps = 383
18:53:12 [INFO] train episode 1040: reward = -33.05, steps = 92
18:53:14 [INFO] train episode 1041: reward = 19.81, steps = 172
18:53:18 [INFO] train episode 1042: reward = 254.22, steps = 332
18:53:19 [INFO] train episode 1043: reward = -31.29, steps = 75
18:53:30 [INFO] train episode 1044: reward = 246.31, steps = 757
18:53:37 [INFO] train episode 1045: reward = 210.33, steps = 495
18:53:51 [INFO] train episode 1046: reward = 117.26, steps = 1000
18:54:05 [INFO] train episode 1047: reward = 129.29, steps = 1000
18:54:20 [INFO] train episode 1048: reward = 142.76, steps = 1000
18:54:21 [INFO] train episode 1049: reward = -51.24, steps = 99
18:54:26 [INFO] train episode 1050: reward = 254.33, steps = 338
18:54:33 [INFO] train episode 1051: reward = 280.18, steps = 516
18:54:35 [INFO] train episode 1052: reward = 221.16, steps = 188
18:54:40 [INFO] train episode 1053: reward = 257.07, steps = 340
18:54:42 [INFO] train episode 1054: reward = -42.04, steps = 129
18:54:44 [INFO] train episode 1055: reward = 18.48, steps = 172
18:54:45 [INFO] train episode 1056: reward = -77.88, steps = 76
18:54:52 [INFO] train episode 1057: reward = 203.52, steps = 481
18:54:53 [INFO] train episode 1058: reward = -45.13, steps = 78
18:55:08 [INFO] train episode 1059: reward = 41.83, steps = 1000
18:55:13 [INFO] train episode 1060: reward = 265.92, steps = 388
18:55:17 [INFO] train episode 1061: reward = 239.85, steps = 258
18:55:19 [INFO] train episode 1062: reward = -23.84, steps = 164
18:55:23 [INFO] train episode 1063: reward = 197.61, steps = 281
18:55:38 [INFO] train episode 1064: reward = 138.22, steps = 1000
18:55:40 [INFO] train episode 1065: reward = -54.58, steps = 128
18:55:42 [INFO] train episode 1066: reward = 17.46, steps = 159
18:55:46 [INFO] train episode 1067: reward = 177.45, steps = 295
18:56:00 [INFO] train episode 1068: reward = 87.30, steps = 1000
18:56:02 [INFO] train episode 1069: reward = -35.31, steps = 108
18:56:05 [INFO] train episode 1070: reward = 245.48, steps = 263
18:56:07 [INFO] train episode 1071: reward = -54.59, steps = 108
18:56:11 [INFO] train episode 1072: reward = 263.13, steps = 312
18:56:14 [INFO] train episode 1073: reward = 253.31, steps = 236
18:56:21 [INFO] train episode 1074: reward = 260.72, steps = 466
18:56:35 [INFO] train episode 1075: reward = 133.72, steps = 1000
18:56:39 [INFO] train episode 1076: reward = 262.66, steps = 251
18:56:40 [INFO] train episode 1077: reward = -89.85, steps = 138
18:56:42 [INFO] train episode 1078: reward = -450.04, steps = 117
18:56:46 [INFO] train episode 1079: reward = 275.63, steps = 274
18:56:48 [INFO] train episode 1080: reward = -65.11, steps = 136
18:57:02 [INFO] train episode 1081: reward = 140.64, steps = 1000
18:57:17 [INFO] train episode 1082: reward = 136.91, steps = 1000
18:57:21 [INFO] train episode 1083: reward = 271.82, steps = 256
18:57:24 [INFO] train episode 1084: reward = 283.92, steps = 272
18:57:26 [INFO] train episode 1085: reward = -60.55, steps = 104
18:57:41 [INFO] train episode 1086: reward = 106.55, steps = 1000
18:57:45 [INFO] train episode 1087: reward = 237.74, steps = 255
18:57:48 [INFO] train episode 1088: reward = 263.52, steps = 211
18:57:51 [INFO] train episode 1089: reward = 232.83, steps = 199
18:57:55 [INFO] train episode 1090: reward = 244.00, steps = 297
18:57:59 [INFO] train episode 1091: reward = 279.19, steps = 262
18:58:03 [INFO] train episode 1092: reward = 203.34, steps = 344
18:58:08 [INFO] train episode 1093: reward = 287.05, steps = 295
18:58:15 [INFO] train episode 1094: reward = 235.25, steps = 491
18:58:19 [INFO] train episode 1095: reward = 250.99, steps = 247
18:58:23 [INFO] train episode 1096: reward = 289.25, steps = 248
18:58:23 [INFO] ==== test ====
18:58:23 [INFO] test episode 0: reward = 279.16, steps = 244
18:58:23 [INFO] test episode 1: reward = 238.80, steps = 277
18:58:24 [INFO] test episode 2: reward = 256.58, steps = 407
18:58:24 [INFO] test episode 3: reward = 260.11, steps = 247
18:58:25 [INFO] test episode 4: reward = 253.51, steps = 295
18:58:25 [INFO] test episode 5: reward = 244.07, steps = 277
18:58:27 [INFO] test episode 6: reward = 107.44, steps = 1000
18:58:28 [INFO] test episode 7: reward = 256.76, steps = 358
18:58:28 [INFO] test episode 8: reward = 253.51, steps = 216
18:58:28 [INFO] test episode 9: reward = 273.04, steps = 241
18:58:28 [INFO] test episode 10: reward = 217.87, steps = 237
18:58:30 [INFO] test episode 11: reward = 104.29, steps = 1000
18:58:31 [INFO] test episode 12: reward = 215.63, steps = 226
18:58:31 [INFO] test episode 13: reward = 260.16, steps = 264
18:58:31 [INFO] test episode 14: reward = 266.09, steps = 357
18:58:32 [INFO] test episode 15: reward = 257.40, steps = 334
18:58:32 [INFO] test episode 16: reward = 246.24, steps = 337
18:58:33 [INFO] test episode 17: reward = 257.08, steps = 353
18:58:33 [INFO] test episode 18: reward = 255.00, steps = 274
18:58:35 [INFO] test episode 19: reward = 151.24, steps = 1000
18:58:36 [INFO] test episode 20: reward = 257.41, steps = 552
18:58:38 [INFO] test episode 21: reward = 107.49, steps = 1000
18:58:38 [INFO] test episode 22: reward = 225.74, steps = 224
18:58:40 [INFO] test episode 23: reward = 141.09, steps = 1000
18:58:42 [INFO] test episode 24: reward = 142.38, steps = 1000
18:58:42 [INFO] test episode 25: reward = 238.57, steps = 257
18:58:42 [INFO] test episode 26: reward = 262.03, steps = 258
18:58:43 [INFO] test episode 27: reward = 253.45, steps = 206
18:58:43 [INFO] test episode 28: reward = 241.28, steps = 345
18:58:43 [INFO] test episode 29: reward = 221.80, steps = 244
18:58:45 [INFO] test episode 30: reward = 158.97, steps = 1000
18:58:46 [INFO] test episode 31: reward = 211.46, steps = 377
18:58:46 [INFO] test episode 32: reward = 263.97, steps = 348
18:58:47 [INFO] test episode 33: reward = 243.19, steps = 256
18:58:47 [INFO] test episode 34: reward = 258.01, steps = 364
18:58:47 [INFO] test episode 35: reward = 279.04, steps = 253
18:58:48 [INFO] test episode 36: reward = 268.28, steps = 285
18:58:48 [INFO] test episode 37: reward = 220.07, steps = 228
18:58:48 [INFO] test episode 38: reward = 253.82, steps = 428
18:58:49 [INFO] test episode 39: reward = 259.66, steps = 309
18:58:51 [INFO] test episode 40: reward = 194.73, steps = 912
18:58:51 [INFO] test episode 41: reward = 266.65, steps = 241
18:58:52 [INFO] test episode 42: reward = 197.65, steps = 513
18:58:52 [INFO] test episode 43: reward = 266.24, steps = 391
18:58:53 [INFO] test episode 44: reward = 253.45, steps = 243
18:58:53 [INFO] test episode 45: reward = 263.09, steps = 365
18:58:55 [INFO] test episode 46: reward = 101.22, steps = 1000
18:58:56 [INFO] test episode 47: reward = 258.37, steps = 471
18:58:56 [INFO] test episode 48: reward = 252.55, steps = 256
18:58:58 [INFO] test episode 49: reward = 168.53, steps = 894
18:58:58 [INFO] test episode 50: reward = 253.71, steps = 439
18:58:59 [INFO] test episode 51: reward = 232.88, steps = 207
18:59:01 [INFO] test episode 52: reward = 210.76, steps = 908
18:59:01 [INFO] test episode 53: reward = 263.08, steps = 343
18:59:02 [INFO] test episode 54: reward = 224.02, steps = 315
18:59:02 [INFO] test episode 55: reward = 232.40, steps = 311
18:59:02 [INFO] test episode 56: reward = 256.63, steps = 305
18:59:03 [INFO] test episode 57: reward = 248.26, steps = 372
18:59:03 [INFO] test episode 58: reward = 273.76, steps = 386
18:59:05 [INFO] test episode 59: reward = 135.52, steps = 1000
18:59:07 [INFO] test episode 60: reward = 148.69, steps = 1000
18:59:07 [INFO] test episode 61: reward = 251.73, steps = 266
18:59:08 [INFO] test episode 62: reward = 251.96, steps = 180
18:59:08 [INFO] test episode 63: reward = 246.56, steps = 313
18:59:08 [INFO] test episode 64: reward = 254.47, steps = 326
18:59:09 [INFO] test episode 65: reward = -2.94, steps = 246
18:59:09 [INFO] test episode 66: reward = 257.42, steps = 354
18:59:09 [INFO] test episode 67: reward = 256.20, steps = 336
18:59:10 [INFO] test episode 68: reward = 246.34, steps = 293
18:59:12 [INFO] test episode 69: reward = 118.35, steps = 1000
18:59:12 [INFO] test episode 70: reward = 221.84, steps = 334
18:59:13 [INFO] test episode 71: reward = 242.60, steps = 349
18:59:13 [INFO] test episode 72: reward = 265.19, steps = 244
18:59:13 [INFO] test episode 73: reward = 272.73, steps = 262
18:59:14 [INFO] test episode 74: reward = 240.03, steps = 443
18:59:14 [INFO] test episode 75: reward = 263.69, steps = 227
18:59:15 [INFO] test episode 76: reward = 195.27, steps = 387
18:59:15 [INFO] test episode 77: reward = 238.01, steps = 432
18:59:16 [INFO] test episode 78: reward = 245.46, steps = 343
18:59:16 [INFO] test episode 79: reward = 237.64, steps = 302
18:59:16 [INFO] test episode 80: reward = 265.55, steps = 278
18:59:17 [INFO] test episode 81: reward = 271.83, steps = 404
18:59:17 [INFO] test episode 82: reward = 255.63, steps = 233
18:59:17 [INFO] test episode 83: reward = 255.71, steps = 434
18:59:18 [INFO] test episode 84: reward = 247.68, steps = 444
18:59:18 [INFO] test episode 85: reward = 202.10, steps = 326
18:59:19 [INFO] test episode 86: reward = 268.52, steps = 255
18:59:19 [INFO] test episode 87: reward = 247.50, steps = 287
18:59:19 [INFO] test episode 88: reward = 276.68, steps = 358
18:59:21 [INFO] test episode 89: reward = 139.25, steps = 1000
18:59:23 [INFO] test episode 90: reward = 94.53, steps = 1000
18:59:23 [INFO] test episode 91: reward = 286.10, steps = 323
18:59:23 [INFO] test episode 92: reward = 263.29, steps = 320
18:59:25 [INFO] test episode 93: reward = 108.22, steps = 1000
18:59:27 [INFO] test episode 94: reward = 146.27, steps = 1000
18:59:27 [INFO] test episode 95: reward = 235.82, steps = 322
18:59:29 [INFO] test episode 96: reward = 143.85, steps = 1000
18:59:30 [INFO] test episode 97: reward = 241.33, steps = 455
18:59:30 [INFO] test episode 98: reward = 287.49, steps = 258
18:59:32 [INFO] test episode 99: reward = 140.57, steps = 1000
18:59:32 [INFO] average episode reward = 225.44 ± 54.79