Use Augmented Random Search to Play BipedalWalker-v3¶

In [1]:
%matplotlib inline

import sys
import logging
import itertools

import numpy as np
np.random.seed(0)
import gym
import matplotlib.pyplot as plt

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')
In [2]:
env = gym.make('BipedalWalker-v3')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

def clip_reward(reward):
    return np.clip(reward, -1., 1.)
reward_clipped_env = gym.wrappers.TransformReward(env, clip_reward)

00:08:42 [INFO] env: <BipedalWalker<BipedalWalker-v3>>
00:08:43 [INFO] action_space: Box(-1.0, 1.0, (4,), float32)
00:08:43 [INFO] observation_space: Box(-inf, inf, (24,), float32)
00:08:43 [INFO] reward_range: (-inf, inf)
00:08:43 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
00:08:43 [INFO] _max_episode_steps: 1600
00:08:43 [INFO] _elapsed_steps: None
00:08:43 [INFO] id: BipedalWalker-v3
00:08:43 [INFO] entry_point: gym.envs.box2d:BipedalWalker
00:08:43 [INFO] reward_threshold: 300
00:08:43 [INFO] nondeterministic: False
00:08:43 [INFO] max_episode_steps: 1600
00:08:43 [INFO] _kwargs: {}
00:08:43 [INFO] _env_name: BipedalWalker
In [3]:
def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps
In [4]:
class ARSAgent:
    def __init__(self, env=None, weights=None, bias=None):
        if weights is not None:
            self.weights = weights
        else:
            self.weights = np.zeros((env.observation_space.shape[0],
                    env.action_space.shape[0]))
        if bias is not None:
            self.bias = bias
        else:
            self.bias = np.zeros(env.action_space.shape[0])

    def reset(self, mode=None):
        pass

    def close(self):
        pass

    def step(self, observation, reward, terminated):
        action = np.matmul(observation, self.weights)
        return action

    def train(self, env, scale=0.06, learning_rate=0.09, population=16):
        weight_updates = np.zeros_like(self.weights)
        bias_updates = np.zeros_like(self.bias)
        for _ in range(population):
            weight_delta = scale * np.random.randn(*agent.weights.shape)
            bias_delta = scale * np.random.randn(*agent.bias.shape)
            pos_agent = ARSAgent(weights=self.weights + weight_delta,
                    bias=self.bias + bias_delta)
            pos_reward, _ = play_episode(env, pos_agent)
            neg_agent = ARSAgent(weights=self.weights - weight_delta,
                    bias=self.bias - bias_delta)
            neg_reward, _ = play_episode(env, neg_agent)
            weight_updates += (pos_reward - neg_reward) * weight_delta
            bias_updates += (pos_reward - neg_reward) * bias_delta
        self.weights += learning_rate * weight_updates / population
        self.bias += learning_rate * bias_updates / population


agent = ARSAgent(env=env)
In [5]:
logging.info('==== train & evaluate ====')
episode_rewards = []
for generation in itertools.count():
    agent.train(reward_clipped_env)
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('evaluate generation %d: reward = %.2f, steps = %d',
            generation, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > 200.:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

00:08:43 [INFO] ==== train & evaluate ====
00:08:55 [INFO] evaluate generation 0: reward = -92.21, steps = 112
00:09:05 [INFO] evaluate generation 1: reward = -92.84, steps = 98
00:09:12 [INFO] evaluate generation 2: reward = -95.17, steps = 67
00:09:15 [INFO] evaluate generation 3: reward = -95.99, steps = 79
00:09:25 [INFO] evaluate generation 4: reward = -93.12, steps = 85
00:09:30 [INFO] evaluate generation 5: reward = -93.49, steps = 86
00:09:40 [INFO] evaluate generation 6: reward = -95.27, steps = 75
00:09:48 [INFO] evaluate generation 7: reward = -97.63, steps = 56
00:09:54 [INFO] evaluate generation 8: reward = -98.29, steps = 57
00:09:57 [INFO] evaluate generation 9: reward = -95.33, steps = 66
00:10:01 [INFO] evaluate generation 10: reward = -98.50, steps = 61
00:10:02 [INFO] evaluate generation 11: reward = -97.95, steps = 65
00:10:06 [INFO] evaluate generation 12: reward = -96.33, steps = 71
00:10:08 [INFO] evaluate generation 13: reward = -101.46, steps = 56
00:10:14 [INFO] evaluate generation 14: reward = -95.11, steps = 77
00:10:15 [INFO] evaluate generation 15: reward = -95.66, steps = 66
00:10:17 [INFO] evaluate generation 16: reward = -94.83, steps = 70
00:10:19 [INFO] evaluate generation 17: reward = -94.91, steps = 84
00:10:21 [INFO] evaluate generation 18: reward = -94.79, steps = 72
00:10:22 [INFO] evaluate generation 19: reward = -95.24, steps = 69
00:10:25 [INFO] evaluate generation 20: reward = -96.11, steps = 63
00:10:26 [INFO] evaluate generation 21: reward = -95.91, steps = 65
00:10:29 [INFO] evaluate generation 22: reward = -97.03, steps = 68
00:10:30 [INFO] evaluate generation 23: reward = -97.11, steps = 60
00:10:31 [INFO] evaluate generation 24: reward = -96.64, steps = 61
00:10:34 [INFO] evaluate generation 25: reward = -97.19, steps = 57
00:10:35 [INFO] evaluate generation 26: reward = -96.58, steps = 60
00:10:37 [INFO] evaluate generation 27: reward = -96.82, steps = 60
00:10:38 [INFO] evaluate generation 28: reward = -95.98, steps = 74
00:10:40 [INFO] evaluate generation 29: reward = -96.40, steps = 61
00:10:43 [INFO] evaluate generation 30: reward = -96.83, steps = 58
00:10:44 [INFO] evaluate generation 31: reward = -96.67, steps = 60
00:10:46 [INFO] evaluate generation 32: reward = -96.42, steps = 60
00:10:47 [INFO] evaluate generation 33: reward = -96.41, steps = 60
00:10:48 [INFO] evaluate generation 34: reward = -96.37, steps = 61
00:10:49 [INFO] evaluate generation 35: reward = -96.56, steps = 59
00:10:51 [INFO] evaluate generation 36: reward = -96.05, steps = 62
00:10:52 [INFO] evaluate generation 37: reward = -96.30, steps = 61
00:10:53 [INFO] evaluate generation 38: reward = -96.37, steps = 61
00:10:55 [INFO] evaluate generation 39: reward = -96.63, steps = 60
00:10:57 [INFO] evaluate generation 40: reward = -96.46, steps = 60
00:10:58 [INFO] evaluate generation 41: reward = -96.32, steps = 60
00:11:00 [INFO] evaluate generation 42: reward = -96.32, steps = 60
00:11:01 [INFO] evaluate generation 43: reward = -96.67, steps = 59
00:11:02 [INFO] evaluate generation 44: reward = -96.37, steps = 60
00:11:03 [INFO] evaluate generation 45: reward = -96.56, steps = 59
00:11:05 [INFO] evaluate generation 46: reward = -96.15, steps = 61
00:11:06 [INFO] evaluate generation 47: reward = -96.01, steps = 62
00:11:09 [INFO] evaluate generation 48: reward = -97.49, steps = 56
00:11:11 [INFO] evaluate generation 49: reward = -97.13, steps = 58
00:11:12 [INFO] evaluate generation 50: reward = -96.51, steps = 60
00:11:14 [INFO] evaluate generation 51: reward = -96.34, steps = 61
00:11:16 [INFO] evaluate generation 52: reward = -96.31, steps = 61
00:11:17 [INFO] evaluate generation 53: reward = -96.68, steps = 60
00:11:19 [INFO] evaluate generation 54: reward = -96.22, steps = 61
00:11:20 [INFO] evaluate generation 55: reward = -95.80, steps = 62
00:11:22 [INFO] evaluate generation 56: reward = -95.26, steps = 65
00:11:24 [INFO] evaluate generation 57: reward = -95.04, steps = 68
00:11:25 [INFO] evaluate generation 58: reward = -94.71, steps = 76
00:11:27 [INFO] evaluate generation 59: reward = -94.31, steps = 80
00:11:28 [INFO] evaluate generation 60: reward = -94.27, steps = 92
00:11:32 [INFO] evaluate generation 61: reward = -96.33, steps = 61
00:11:34 [INFO] evaluate generation 62: reward = -96.10, steps = 62
00:11:35 [INFO] evaluate generation 63: reward = -97.89, steps = 58
00:11:38 [INFO] evaluate generation 64: reward = -95.06, steps = 65
00:11:40 [INFO] evaluate generation 65: reward = -95.90, steps = 63
00:11:42 [INFO] evaluate generation 66: reward = -95.79, steps = 63
00:11:45 [INFO] evaluate generation 67: reward = -97.26, steps = 60
00:11:47 [INFO] evaluate generation 68: reward = -96.28, steps = 61
00:11:48 [INFO] evaluate generation 69: reward = -96.86, steps = 60
00:11:50 [INFO] evaluate generation 70: reward = -97.71, steps = 59
00:11:52 [INFO] evaluate generation 71: reward = -95.50, steps = 63
00:11:53 [INFO] evaluate generation 72: reward = -94.78, steps = 72
00:11:55 [INFO] evaluate generation 73: reward = -94.93, steps = 72
00:11:57 [INFO] evaluate generation 74: reward = -94.64, steps = 72
00:12:01 [INFO] evaluate generation 75: reward = -96.15, steps = 61
00:12:03 [INFO] evaluate generation 76: reward = -96.67, steps = 59
00:12:04 [INFO] evaluate generation 77: reward = -96.71, steps = 59
00:12:06 [INFO] evaluate generation 78: reward = -96.60, steps = 61
00:12:07 [INFO] evaluate generation 79: reward = -95.16, steps = 66
00:12:11 [INFO] evaluate generation 80: reward = -96.25, steps = 61
00:12:12 [INFO] evaluate generation 81: reward = -97.80, steps = 58
00:12:15 [INFO] evaluate generation 82: reward = -96.33, steps = 61
00:12:16 [INFO] evaluate generation 83: reward = -98.66, steps = 55
00:12:18 [INFO] evaluate generation 84: reward = -98.18, steps = 55
00:12:19 [INFO] evaluate generation 85: reward = -97.38, steps = 59
00:12:21 [INFO] evaluate generation 86: reward = -95.88, steps = 61
00:12:23 [INFO] evaluate generation 87: reward = -98.22, steps = 56
00:12:25 [INFO] evaluate generation 88: reward = -96.64, steps = 59
00:12:28 [INFO] evaluate generation 89: reward = -96.33, steps = 60
00:12:30 [INFO] evaluate generation 90: reward = -98.53, steps = 54
00:12:33 [INFO] evaluate generation 91: reward = -96.55, steps = 59
00:12:34 [INFO] evaluate generation 92: reward = -96.47, steps = 60
00:12:35 [INFO] evaluate generation 93: reward = -96.34, steps = 60
00:12:37 [INFO] evaluate generation 94: reward = -98.64, steps = 55
00:12:38 [INFO] evaluate generation 95: reward = -97.60, steps = 57
00:12:40 [INFO] evaluate generation 96: reward = -96.51, steps = 59
00:12:42 [INFO] evaluate generation 97: reward = -97.10, steps = 58
00:12:44 [INFO] evaluate generation 98: reward = -97.64, steps = 57
00:12:45 [INFO] evaluate generation 99: reward = -96.19, steps = 61
00:12:46 [INFO] evaluate generation 100: reward = -95.32, steps = 61
00:12:48 [INFO] evaluate generation 101: reward = -95.00, steps = 63
00:12:50 [INFO] evaluate generation 102: reward = -94.94, steps = 63
00:12:52 [INFO] evaluate generation 103: reward = -95.01, steps = 64
00:12:55 [INFO] evaluate generation 104: reward = -96.01, steps = 60
00:12:57 [INFO] evaluate generation 105: reward = -96.23, steps = 59
00:12:58 [INFO] evaluate generation 106: reward = -95.57, steps = 61
00:13:00 [INFO] evaluate generation 107: reward = -95.34, steps = 63
00:13:01 [INFO] evaluate generation 108: reward = -95.27, steps = 64
00:13:03 [INFO] evaluate generation 109: reward = -95.10, steps = 66
00:13:04 [INFO] evaluate generation 110: reward = -97.08, steps = 59
00:13:06 [INFO] evaluate generation 111: reward = -95.23, steps = 67
00:13:08 [INFO] evaluate generation 112: reward = -94.88, steps = 67
00:13:09 [INFO] evaluate generation 113: reward = -94.77, steps = 72
00:13:11 [INFO] evaluate generation 114: reward = -94.83, steps = 73
00:13:13 [INFO] evaluate generation 115: reward = -95.51, steps = 61
00:13:15 [INFO] evaluate generation 116: reward = -95.40, steps = 63
00:13:16 [INFO] evaluate generation 117: reward = -95.14, steps = 69
00:13:18 [INFO] evaluate generation 118: reward = -95.76, steps = 63
00:13:20 [INFO] evaluate generation 119: reward = -95.02, steps = 66
00:13:21 [INFO] evaluate generation 120: reward = -94.87, steps = 70
00:13:23 [INFO] evaluate generation 121: reward = -94.73, steps = 72
00:13:24 [INFO] evaluate generation 122: reward = -94.16, steps = 80
00:13:27 [INFO] evaluate generation 123: reward = -93.68, steps = 79
00:13:29 [INFO] evaluate generation 124: reward = -94.27, steps = 68
00:13:31 [INFO] evaluate generation 125: reward = -94.68, steps = 70
00:13:32 [INFO] evaluate generation 126: reward = -93.89, steps = 77
00:13:34 [INFO] evaluate generation 127: reward = -93.96, steps = 78
00:13:37 [INFO] evaluate generation 128: reward = -95.00, steps = 69
00:13:38 [INFO] evaluate generation 129: reward = -94.10, steps = 74
00:13:40 [INFO] evaluate generation 130: reward = -94.21, steps = 77
00:13:43 [INFO] evaluate generation 131: reward = -93.97, steps = 70
00:13:45 [INFO] evaluate generation 132: reward = -95.44, steps = 62
00:13:46 [INFO] evaluate generation 133: reward = -94.20, steps = 65
00:13:48 [INFO] evaluate generation 134: reward = -94.35, steps = 74
00:13:50 [INFO] evaluate generation 135: reward = -93.54, steps = 78
00:13:53 [INFO] evaluate generation 136: reward = -93.88, steps = 75
00:13:55 [INFO] evaluate generation 137: reward = -94.70, steps = 67
00:13:56 [INFO] evaluate generation 138: reward = -94.56, steps = 69
00:13:59 [INFO] evaluate generation 139: reward = -95.06, steps = 63
00:14:00 [INFO] evaluate generation 140: reward = -95.21, steps = 64
00:14:02 [INFO] evaluate generation 141: reward = -94.77, steps = 67
00:14:04 [INFO] evaluate generation 142: reward = -95.40, steps = 62
00:14:06 [INFO] evaluate generation 143: reward = -95.40, steps = 63
00:14:08 [INFO] evaluate generation 144: reward = -94.87, steps = 66
00:14:09 [INFO] evaluate generation 145: reward = -94.60, steps = 68
00:14:11 [INFO] evaluate generation 146: reward = -95.12, steps = 67
00:14:13 [INFO] evaluate generation 147: reward = -94.26, steps = 73
00:14:14 [INFO] evaluate generation 148: reward = -94.33, steps = 74
00:14:16 [INFO] evaluate generation 149: reward = -94.36, steps = 76
00:14:17 [INFO] evaluate generation 150: reward = -94.21, steps = 78
00:14:19 [INFO] evaluate generation 151: reward = -94.52, steps = 75
00:14:20 [INFO] evaluate generation 152: reward = -94.29, steps = 75
00:14:22 [INFO] evaluate generation 153: reward = -94.08, steps = 73
00:14:25 [INFO] evaluate generation 154: reward = -94.59, steps = 77
00:14:26 [INFO] evaluate generation 155: reward = -94.65, steps = 79
00:14:28 [INFO] evaluate generation 156: reward = -94.53, steps = 80
00:14:30 [INFO] evaluate generation 157: reward = -94.24, steps = 74
00:14:31 [INFO] evaluate generation 158: reward = -94.04, steps = 75
00:14:33 [INFO] evaluate generation 159: reward = -94.46, steps = 72
00:14:35 [INFO] evaluate generation 160: reward = -94.38, steps = 72
00:14:37 [INFO] evaluate generation 161: reward = -94.26, steps = 73
00:14:38 [INFO] evaluate generation 162: reward = -94.59, steps = 72
00:14:40 [INFO] evaluate generation 163: reward = -94.16, steps = 73
00:14:42 [INFO] evaluate generation 164: reward = -94.59, steps = 73
00:14:43 [INFO] evaluate generation 165: reward = -94.27, steps = 73
00:14:46 [INFO] evaluate generation 166: reward = -93.82, steps = 77
00:14:47 [INFO] evaluate generation 167: reward = -93.85, steps = 76
00:14:51 [INFO] evaluate generation 168: reward = -95.21, steps = 78
00:14:52 [INFO] evaluate generation 169: reward = -95.14, steps = 84
00:14:54 [INFO] evaluate generation 170: reward = -95.23, steps = 70
00:14:56 [INFO] evaluate generation 171: reward = -94.82, steps = 81
00:14:57 [INFO] evaluate generation 172: reward = -94.26, steps = 84
00:14:59 [INFO] evaluate generation 173: reward = -93.89, steps = 74
00:15:02 [INFO] evaluate generation 174: reward = -95.51, steps = 64
00:15:04 [INFO] evaluate generation 175: reward = -95.57, steps = 68
00:15:05 [INFO] evaluate generation 176: reward = -95.06, steps = 74
00:15:07 [INFO] evaluate generation 177: reward = -94.93, steps = 80
00:15:08 [INFO] evaluate generation 178: reward = -94.71, steps = 81
00:15:10 [INFO] evaluate generation 179: reward = -94.19, steps = 78
00:15:13 [INFO] evaluate generation 180: reward = -95.45, steps = 66
00:15:15 [INFO] evaluate generation 181: reward = -95.45, steps = 62
00:15:17 [INFO] evaluate generation 182: reward = -94.82, steps = 64
00:15:18 [INFO] evaluate generation 183: reward = -94.57, steps = 66
00:15:20 [INFO] evaluate generation 184: reward = -96.60, steps = 60
00:15:21 [INFO] evaluate generation 185: reward = -97.05, steps = 59
00:15:22 [INFO] evaluate generation 186: reward = -95.44, steps = 62
00:15:24 [INFO] evaluate generation 187: reward = -96.32, steps = 61
00:15:25 [INFO] evaluate generation 188: reward = -96.45, steps = 61
00:15:27 [INFO] evaluate generation 189: reward = -95.30, steps = 62
00:15:29 [INFO] evaluate generation 190: reward = -95.31, steps = 62
00:15:30 [INFO] evaluate generation 191: reward = -95.37, steps = 62
00:15:31 [INFO] evaluate generation 192: reward = -95.92, steps = 61
00:15:33 [INFO] evaluate generation 193: reward = -95.77, steps = 62
00:15:34 [INFO] evaluate generation 194: reward = -95.47, steps = 62
00:15:36 [INFO] evaluate generation 195: reward = -94.82, steps = 64
00:15:37 [INFO] evaluate generation 196: reward = -94.44, steps = 68
00:15:39 [INFO] evaluate generation 197: reward = -94.58, steps = 68
00:15:41 [INFO] evaluate generation 198: reward = -94.77, steps = 66
00:15:42 [INFO] evaluate generation 199: reward = -95.37, steps = 63
00:15:44 [INFO] evaluate generation 200: reward = -95.44, steps = 63
00:15:45 [INFO] evaluate generation 201: reward = -95.32, steps = 62
00:15:47 [INFO] evaluate generation 202: reward = -94.59, steps = 66
00:15:48 [INFO] evaluate generation 203: reward = -94.38, steps = 68
00:15:51 [INFO] evaluate generation 204: reward = -95.68, steps = 63
00:15:52 [INFO] evaluate generation 205: reward = -95.43, steps = 62
00:15:53 [INFO] evaluate generation 206: reward = -95.45, steps = 62
00:15:55 [INFO] evaluate generation 207: reward = -94.71, steps = 65
00:15:56 [INFO] evaluate generation 208: reward = -94.29, steps = 67
00:15:58 [INFO] evaluate generation 209: reward = -94.44, steps = 68
00:16:00 [INFO] evaluate generation 210: reward = -94.43, steps = 69
00:16:02 [INFO] evaluate generation 211: reward = -94.47, steps = 69
00:16:03 [INFO] evaluate generation 212: reward = -94.31, steps = 70
00:16:05 [INFO] evaluate generation 213: reward = -94.41, steps = 70
00:16:06 [INFO] evaluate generation 214: reward = -94.06, steps = 72
00:16:08 [INFO] evaluate generation 215: reward = -93.91, steps = 77
00:16:09 [INFO] evaluate generation 216: reward = -93.98, steps = 77
00:16:12 [INFO] evaluate generation 217: reward = -94.67, steps = 71
00:16:13 [INFO] evaluate generation 218: reward = -94.47, steps = 73
00:16:15 [INFO] evaluate generation 219: reward = -94.40, steps = 73
00:16:18 [INFO] evaluate generation 220: reward = -95.47, steps = 70
00:16:19 [INFO] evaluate generation 221: reward = -94.99, steps = 70
00:16:21 [INFO] evaluate generation 222: reward = -95.05, steps = 73
00:16:23 [INFO] evaluate generation 223: reward = -94.68, steps = 77
00:16:24 [INFO] evaluate generation 224: reward = -95.11, steps = 79
00:16:26 [INFO] evaluate generation 225: reward = -95.04, steps = 77
00:16:28 [INFO] evaluate generation 226: reward = -94.86, steps = 71
00:16:29 [INFO] evaluate generation 227: reward = -95.01, steps = 73
00:16:31 [INFO] evaluate generation 228: reward = -94.71, steps = 73
00:16:32 [INFO] evaluate generation 229: reward = -94.58, steps = 77
00:16:35 [INFO] evaluate generation 230: reward = -94.27, steps = 68
00:16:37 [INFO] evaluate generation 231: reward = -94.34, steps = 70
00:16:38 [INFO] evaluate generation 232: reward = -93.96, steps = 69
00:16:40 [INFO] evaluate generation 233: reward = -94.00, steps = 70
00:16:41 [INFO] evaluate generation 234: reward = -94.14, steps = 69
00:16:43 [INFO] evaluate generation 235: reward = -94.03, steps = 71
00:16:44 [INFO] evaluate generation 236: reward = -94.03, steps = 71
00:16:47 [INFO] evaluate generation 237: reward = -95.40, steps = 62
00:16:48 [INFO] evaluate generation 238: reward = -94.44, steps = 65
00:16:51 [INFO] evaluate generation 239: reward = -95.37, steps = 63
00:16:52 [INFO] evaluate generation 240: reward = -95.19, steps = 63
00:16:54 [INFO] evaluate generation 241: reward = -95.13, steps = 63
00:16:55 [INFO] evaluate generation 242: reward = -94.77, steps = 64
00:16:56 [INFO] evaluate generation 243: reward = -93.87, steps = 67
00:16:58 [INFO] evaluate generation 244: reward = -93.33, steps = 69
00:17:00 [INFO] evaluate generation 245: reward = -94.96, steps = 63
00:17:02 [INFO] evaluate generation 246: reward = -94.44, steps = 65
00:17:03 [INFO] evaluate generation 247: reward = -93.92, steps = 67
00:17:05 [INFO] evaluate generation 248: reward = -93.42, steps = 70
00:17:06 [INFO] evaluate generation 249: reward = -93.56, steps = 69
00:17:08 [INFO] evaluate generation 250: reward = -94.13, steps = 66
00:17:09 [INFO] evaluate generation 251: reward = -93.54, steps = 70
00:17:11 [INFO] evaluate generation 252: reward = -95.31, steps = 65
00:17:13 [INFO] evaluate generation 253: reward = -94.50, steps = 66
00:17:14 [INFO] evaluate generation 254: reward = -94.37, steps = 67
00:17:16 [INFO] evaluate generation 255: reward = -94.08, steps = 70
00:17:17 [INFO] evaluate generation 256: reward = -96.29, steps = 63
00:17:19 [INFO] evaluate generation 257: reward = -94.44, steps = 65
00:17:20 [INFO] evaluate generation 258: reward = -93.96, steps = 68
00:17:22 [INFO] evaluate generation 259: reward = -94.13, steps = 68
00:17:24 [INFO] evaluate generation 260: reward = -93.89, steps = 72
00:17:25 [INFO] evaluate generation 261: reward = -94.49, steps = 66
00:17:27 [INFO] evaluate generation 262: reward = -94.11, steps = 68
00:17:28 [INFO] evaluate generation 263: reward = -94.25, steps = 67
00:17:30 [INFO] evaluate generation 264: reward = -94.67, steps = 64
00:17:31 [INFO] evaluate generation 265: reward = -95.62, steps = 64
00:17:33 [INFO] evaluate generation 266: reward = -94.36, steps = 68
00:17:35 [INFO] evaluate generation 267: reward = -95.35, steps = 63
00:17:37 [INFO] evaluate generation 268: reward = -95.24, steps = 66
00:17:39 [INFO] evaluate generation 269: reward = -95.16, steps = 66
00:17:40 [INFO] evaluate generation 270: reward = -95.04, steps = 66
00:17:42 [INFO] evaluate generation 271: reward = -94.83, steps = 67
00:17:43 [INFO] evaluate generation 272: reward = -94.86, steps = 67
00:17:45 [INFO] evaluate generation 273: reward = -96.36, steps = 63
00:17:46 [INFO] evaluate generation 274: reward = -96.82, steps = 62
00:17:48 [INFO] evaluate generation 275: reward = -94.53, steps = 68
00:17:50 [INFO] evaluate generation 276: reward = -95.33, steps = 64
00:17:51 [INFO] evaluate generation 277: reward = -94.69, steps = 66
00:17:53 [INFO] evaluate generation 278: reward = -94.60, steps = 67
00:17:54 [INFO] evaluate generation 279: reward = -96.39, steps = 62
00:17:56 [INFO] evaluate generation 280: reward = -93.24, steps = 73
00:17:58 [INFO] evaluate generation 281: reward = -94.55, steps = 66
00:17:59 [INFO] evaluate generation 282: reward = -94.32, steps = 67
00:18:01 [INFO] evaluate generation 283: reward = -92.84, steps = 74
00:18:03 [INFO] evaluate generation 284: reward = -94.65, steps = 66
00:18:05 [INFO] evaluate generation 285: reward = -94.60, steps = 66
00:18:06 [INFO] evaluate generation 286: reward = -93.80, steps = 69
00:18:08 [INFO] evaluate generation 287: reward = -95.20, steps = 65
00:18:09 [INFO] evaluate generation 288: reward = -94.70, steps = 65
00:18:11 [INFO] evaluate generation 289: reward = -92.66, steps = 75
00:18:12 [INFO] evaluate generation 290: reward = -93.03, steps = 72
00:18:14 [INFO] evaluate generation 291: reward = -92.56, steps = 75
00:18:16 [INFO] evaluate generation 292: reward = -93.24, steps = 70
00:18:18 [INFO] evaluate generation 293: reward = -92.57, steps = 77
00:18:20 [INFO] evaluate generation 294: reward = -92.31, steps = 78
00:18:22 [INFO] evaluate generation 295: reward = -92.46, steps = 77
00:18:24 [INFO] evaluate generation 296: reward = -92.62, steps = 79
00:18:26 [INFO] evaluate generation 297: reward = -92.91, steps = 75
00:18:28 [INFO] evaluate generation 298: reward = -93.18, steps = 72
00:18:30 [INFO] evaluate generation 299: reward = -92.94, steps = 73
00:18:31 [INFO] evaluate generation 300: reward = -93.35, steps = 70
00:18:33 [INFO] evaluate generation 301: reward = -93.30, steps = 70
00:18:35 [INFO] evaluate generation 302: reward = -92.73, steps = 74
00:18:38 [INFO] evaluate generation 303: reward = -93.95, steps = 67
00:18:40 [INFO] evaluate generation 304: reward = -94.08, steps = 67
00:18:41 [INFO] evaluate generation 305: reward = -93.72, steps = 69
00:18:43 [INFO] evaluate generation 306: reward = -93.50, steps = 69
00:18:47 [INFO] evaluate generation 307: reward = -94.92, steps = 65
00:18:49 [INFO] evaluate generation 308: reward = -95.01, steps = 64
00:18:50 [INFO] evaluate generation 309: reward = -94.76, steps = 66
00:18:52 [INFO] evaluate generation 310: reward = -93.27, steps = 72
00:18:53 [INFO] evaluate generation 311: reward = -94.00, steps = 69
00:18:55 [INFO] evaluate generation 312: reward = -93.47, steps = 70
00:18:56 [INFO] evaluate generation 313: reward = -93.88, steps = 69
00:18:59 [INFO] evaluate generation 314: reward = -93.13, steps = 73
00:19:01 [INFO] evaluate generation 315: reward = -93.25, steps = 74
00:19:02 [INFO] evaluate generation 316: reward = -93.00, steps = 76
00:19:05 [INFO] evaluate generation 317: reward = -93.47, steps = 73
00:19:08 [INFO] evaluate generation 318: reward = -93.64, steps = 72
00:19:10 [INFO] evaluate generation 319: reward = -93.65, steps = 73
00:19:15 [INFO] evaluate generation 320: reward = -94.59, steps = 67
00:19:17 [INFO] evaluate generation 321: reward = -94.50, steps = 68
00:19:20 [INFO] evaluate generation 322: reward = -94.56, steps = 70
00:19:22 [INFO] evaluate generation 323: reward = -93.08, steps = 74
00:19:24 [INFO] evaluate generation 324: reward = -93.01, steps = 74
00:19:26 [INFO] evaluate generation 325: reward = -93.08, steps = 74
00:19:28 [INFO] evaluate generation 326: reward = -92.74, steps = 75
00:19:30 [INFO] evaluate generation 327: reward = -93.33, steps = 72
00:19:32 [INFO] evaluate generation 328: reward = -92.40, steps = 77
00:19:33 [INFO] evaluate generation 329: reward = -92.62, steps = 75
00:19:35 [INFO] evaluate generation 330: reward = -93.00, steps = 73
00:19:37 [INFO] evaluate generation 331: reward = -92.50, steps = 77
00:19:38 [INFO] evaluate generation 332: reward = -91.74, steps = 84
00:19:41 [INFO] evaluate generation 333: reward = -93.22, steps = 74
00:19:44 [INFO] evaluate generation 334: reward = -93.53, steps = 72
00:19:46 [INFO] evaluate generation 335: reward = -93.73, steps = 72
00:19:50 [INFO] evaluate generation 336: reward = -94.11, steps = 71
00:19:52 [INFO] evaluate generation 337: reward = -92.66, steps = 76
00:19:54 [INFO] evaluate generation 338: reward = -92.17, steps = 80
00:19:56 [INFO] evaluate generation 339: reward = -92.76, steps = 80
00:19:58 [INFO] evaluate generation 340: reward = -92.09, steps = 82
00:20:00 [INFO] evaluate generation 341: reward = -92.82, steps = 77
00:20:02 [INFO] evaluate generation 342: reward = -91.99, steps = 80
00:20:04 [INFO] evaluate generation 343: reward = -91.48, steps = 80
00:20:05 [INFO] evaluate generation 344: reward = -91.43, steps = 80
00:20:07 [INFO] evaluate generation 345: reward = -92.42, steps = 82
00:20:10 [INFO] evaluate generation 346: reward = -91.45, steps = 80
00:20:11 [INFO] evaluate generation 347: reward = -90.79, steps = 83
00:20:13 [INFO] evaluate generation 348: reward = -89.91, steps = 88
00:20:15 [INFO] evaluate generation 349: reward = -89.92, steps = 90
00:20:17 [INFO] evaluate generation 350: reward = -88.97, steps = 100
00:20:20 [INFO] evaluate generation 351: reward = -92.78, steps = 82
00:20:24 [INFO] evaluate generation 352: reward = -92.42, steps = 81
00:20:25 [INFO] evaluate generation 353: reward = -92.21, steps = 84
00:20:28 [INFO] evaluate generation 354: reward = -92.18, steps = 85
00:20:30 [INFO] evaluate generation 355: reward = -92.84, steps = 76
00:20:31 [INFO] evaluate generation 356: reward = -91.45, steps = 86
00:20:34 [INFO] evaluate generation 357: reward = -93.04, steps = 76
00:20:36 [INFO] evaluate generation 358: reward = -92.83, steps = 76
00:20:37 [INFO] evaluate generation 359: reward = -92.42, steps = 80
00:20:40 [INFO] evaluate generation 360: reward = -91.71, steps = 86
00:20:43 [INFO] evaluate generation 361: reward = -93.92, steps = 71
00:20:46 [INFO] evaluate generation 362: reward = -94.12, steps = 70
00:20:48 [INFO] evaluate generation 363: reward = -93.92, steps = 70
00:20:50 [INFO] evaluate generation 364: reward = -93.38, steps = 75
00:20:51 [INFO] evaluate generation 365: reward = -93.71, steps = 73
00:20:53 [INFO] evaluate generation 366: reward = -93.30, steps = 80
00:20:57 [INFO] evaluate generation 367: reward = -93.21, steps = 73
00:20:59 [INFO] evaluate generation 368: reward = -93.06, steps = 76
00:21:01 [INFO] evaluate generation 369: reward = -93.35, steps = 74
00:21:03 [INFO] evaluate generation 370: reward = -92.56, steps = 78
00:21:05 [INFO] evaluate generation 371: reward = -93.04, steps = 78
00:21:07 [INFO] evaluate generation 372: reward = -92.26, steps = 77
00:21:10 [INFO] evaluate generation 373: reward = -94.00, steps = 70
00:21:11 [INFO] evaluate generation 374: reward = -92.96, steps = 81
00:21:13 [INFO] evaluate generation 375: reward = -92.62, steps = 79
00:21:15 [INFO] evaluate generation 376: reward = -92.16, steps = 77
00:21:17 [INFO] evaluate generation 377: reward = -92.77, steps = 76
00:21:20 [INFO] evaluate generation 378: reward = -92.81, steps = 81
00:21:21 [INFO] evaluate generation 379: reward = -92.38, steps = 77
00:21:24 [INFO] evaluate generation 380: reward = -93.91, steps = 72
00:21:25 [INFO] evaluate generation 381: reward = -93.03, steps = 78
00:21:27 [INFO] evaluate generation 382: reward = -93.46, steps = 73
00:21:29 [INFO] evaluate generation 383: reward = -93.32, steps = 74
00:21:31 [INFO] evaluate generation 384: reward = -92.69, steps = 78
00:21:33 [INFO] evaluate generation 385: reward = -92.69, steps = 78
00:21:34 [INFO] evaluate generation 386: reward = -92.21, steps = 84
00:21:36 [INFO] evaluate generation 387: reward = -93.12, steps = 76
00:21:38 [INFO] evaluate generation 388: reward = -92.24, steps = 84
00:21:39 [INFO] evaluate generation 389: reward = -93.34, steps = 76
00:21:41 [INFO] evaluate generation 390: reward = -94.03, steps = 75
00:21:43 [INFO] evaluate generation 391: reward = -93.13, steps = 80
00:21:44 [INFO] evaluate generation 392: reward = -92.58, steps = 84
00:21:46 [INFO] evaluate generation 393: reward = -93.03, steps = 78
00:21:48 [INFO] evaluate generation 394: reward = -91.51, steps = 86
00:21:50 [INFO] evaluate generation 395: reward = -92.65, steps = 79
00:21:52 [INFO] evaluate generation 396: reward = -92.20, steps = 82
00:21:56 [INFO] evaluate generation 397: reward = -93.10, steps = 77
00:21:58 [INFO] evaluate generation 398: reward = -92.30, steps = 76
00:22:00 [INFO] evaluate generation 399: reward = -92.45, steps = 76
00:22:01 [INFO] evaluate generation 400: reward = -92.13, steps = 80
00:22:05 [INFO] evaluate generation 401: reward = -93.04, steps = 74
00:22:07 [INFO] evaluate generation 402: reward = -92.31, steps = 78
00:22:08 [INFO] evaluate generation 403: reward = -92.26, steps = 77
00:22:10 [INFO] evaluate generation 404: reward = -92.07, steps = 81
00:22:12 [INFO] evaluate generation 405: reward = -92.82, steps = 77
00:22:15 [INFO] evaluate generation 406: reward = -92.93, steps = 76
00:22:17 [INFO] evaluate generation 407: reward = -92.50, steps = 76
00:22:18 [INFO] evaluate generation 408: reward = -92.18, steps = 78
00:22:23 [INFO] evaluate generation 409: reward = -93.91, steps = 73
00:22:25 [INFO] evaluate generation 410: reward = -93.94, steps = 74
00:22:26 [INFO] evaluate generation 411: reward = -93.49, steps = 83
00:22:28 [INFO] evaluate generation 412: reward = -93.23, steps = 84
00:22:30 [INFO] evaluate generation 413: reward = -93.43, steps = 83
00:22:31 [INFO] evaluate generation 414: reward = -93.43, steps = 88
00:22:34 [INFO] evaluate generation 415: reward = -93.08, steps = 83
00:22:36 [INFO] evaluate generation 416: reward = -93.24, steps = 80
00:22:38 [INFO] evaluate generation 417: reward = -92.73, steps = 82
00:22:40 [INFO] evaluate generation 418: reward = -92.67, steps = 86
00:22:41 [INFO] evaluate generation 419: reward = -92.67, steps = 87
00:22:43 [INFO] evaluate generation 420: reward = -92.49, steps = 84
00:22:45 [INFO] evaluate generation 421: reward = -92.79, steps = 79
00:22:47 [INFO] evaluate generation 422: reward = -93.06, steps = 83
00:22:49 [INFO] evaluate generation 423: reward = -92.83, steps = 91
00:22:51 [INFO] evaluate generation 424: reward = -92.65, steps = 82
00:22:54 [INFO] evaluate generation 425: reward = -92.87, steps = 82
00:22:56 [INFO] evaluate generation 426: reward = -92.85, steps = 80
00:22:58 [INFO] evaluate generation 427: reward = -92.65, steps = 78
00:22:59 [INFO] evaluate generation 428: reward = -92.30, steps = 83
00:23:01 [INFO] evaluate generation 429: reward = -91.94, steps = 85
00:23:03 [INFO] evaluate generation 430: reward = -92.33, steps = 79
00:23:06 [INFO] evaluate generation 431: reward = -93.55, steps = 74
00:23:08 [INFO] evaluate generation 432: reward = -94.08, steps = 71
00:23:09 [INFO] evaluate generation 433: reward = -92.92, steps = 75
00:23:11 [INFO] evaluate generation 434: reward = -92.02, steps = 83
00:23:12 [INFO] evaluate generation 435: reward = -93.06, steps = 76
00:23:14 [INFO] evaluate generation 436: reward = -92.82, steps = 78
00:23:16 [INFO] evaluate generation 437: reward = -92.32, steps = 82
00:23:18 [INFO] evaluate generation 438: reward = -90.83, steps = 84
00:23:20 [INFO] evaluate generation 439: reward = -92.19, steps = 85
00:23:22 [INFO] evaluate generation 440: reward = -92.14, steps = 88
00:23:24 [INFO] evaluate generation 441: reward = -89.61, steps = 91
00:23:28 [INFO] evaluate generation 442: reward = -91.92, steps = 84
00:23:31 [INFO] evaluate generation 443: reward = -92.29, steps = 82
00:23:33 [INFO] evaluate generation 444: reward = -90.76, steps = 86
00:23:36 [INFO] evaluate generation 445: reward = -92.69, steps = 77
00:23:39 [INFO] evaluate generation 446: reward = -92.59, steps = 77
00:23:40 [INFO] evaluate generation 447: reward = -92.49, steps = 81
00:23:43 [INFO] evaluate generation 448: reward = -92.57, steps = 77
00:23:44 [INFO] evaluate generation 449: reward = -92.89, steps = 74
00:23:46 [INFO] evaluate generation 450: reward = -92.66, steps = 77
00:23:48 [INFO] evaluate generation 451: reward = -92.12, steps = 80
00:23:49 [INFO] evaluate generation 452: reward = -92.25, steps = 81
00:23:52 [INFO] evaluate generation 453: reward = -92.51, steps = 78
00:23:53 [INFO] evaluate generation 454: reward = -93.44, steps = 74
00:23:56 [INFO] evaluate generation 455: reward = -92.39, steps = 79
00:23:58 [INFO] evaluate generation 456: reward = -93.64, steps = 72
00:23:59 [INFO] evaluate generation 457: reward = -92.22, steps = 84
00:24:02 [INFO] evaluate generation 458: reward = -93.09, steps = 75
00:24:04 [INFO] evaluate generation 459: reward = -92.26, steps = 80
00:24:05 [INFO] evaluate generation 460: reward = -92.57, steps = 78
00:24:07 [INFO] evaluate generation 461: reward = -91.05, steps = 90
00:24:09 [INFO] evaluate generation 462: reward = -93.11, steps = 77
00:24:11 [INFO] evaluate generation 463: reward = -89.81, steps = 86
00:24:14 [INFO] evaluate generation 464: reward = -92.67, steps = 82
00:24:17 [INFO] evaluate generation 465: reward = -91.59, steps = 88
00:24:19 [INFO] evaluate generation 466: reward = -91.64, steps = 85
00:24:22 [INFO] evaluate generation 467: reward = -91.64, steps = 85
00:24:25 [INFO] evaluate generation 468: reward = -92.01, steps = 82
00:24:30 [INFO] evaluate generation 469: reward = -91.70, steps = 83
00:24:31 [INFO] evaluate generation 470: reward = -91.81, steps = 85
00:24:33 [INFO] evaluate generation 471: reward = -92.36, steps = 83
00:24:35 [INFO] evaluate generation 472: reward = -91.32, steps = 88
00:24:37 [INFO] evaluate generation 473: reward = -90.45, steps = 91
00:24:40 [INFO] evaluate generation 474: reward = -90.49, steps = 89
00:24:42 [INFO] evaluate generation 475: reward = -90.79, steps = 87
00:24:44 [INFO] evaluate generation 476: reward = -89.41, steps = 89
00:24:46 [INFO] evaluate generation 477: reward = -88.61, steps = 100
00:24:49 [INFO] evaluate generation 478: reward = -92.67, steps = 79
00:24:51 [INFO] evaluate generation 479: reward = -90.55, steps = 86
00:24:57 [INFO] evaluate generation 480: reward = -94.30, steps = 67
00:24:58 [INFO] evaluate generation 481: reward = -94.38, steps = 69
00:25:00 [INFO] evaluate generation 482: reward = -93.01, steps = 75
00:25:01 [INFO] evaluate generation 483: reward = -92.05, steps = 81
00:25:03 [INFO] evaluate generation 484: reward = -91.83, steps = 82
00:25:05 [INFO] evaluate generation 485: reward = -91.43, steps = 83
00:25:06 [INFO] evaluate generation 486: reward = -93.30, steps = 75
00:25:09 [INFO] evaluate generation 487: reward = -91.92, steps = 80
00:25:10 [INFO] evaluate generation 488: reward = -90.03, steps = 85
00:25:12 [INFO] evaluate generation 489: reward = -91.01, steps = 83
00:25:14 [INFO] evaluate generation 490: reward = -88.99, steps = 94
00:25:16 [INFO] evaluate generation 491: reward = -88.14, steps = 102
00:25:19 [INFO] evaluate generation 492: reward = -87.99, steps = 98
00:25:22 [INFO] evaluate generation 493: reward = -88.24, steps = 98
00:25:25 [INFO] evaluate generation 494: reward = -89.65, steps = 88
00:25:27 [INFO] evaluate generation 495: reward = -88.19, steps = 97
00:25:31 [INFO] evaluate generation 496: reward = -90.23, steps = 85
00:25:34 [INFO] evaluate generation 497: reward = -90.32, steps = 84
00:25:35 [INFO] evaluate generation 498: reward = -89.24, steps = 87
00:25:37 [INFO] evaluate generation 499: reward = -92.59, steps = 82
00:25:40 [INFO] evaluate generation 500: reward = -90.66, steps = 82
00:25:42 [INFO] evaluate generation 501: reward = -89.09, steps = 87
00:25:44 [INFO] evaluate generation 502: reward = -89.12, steps = 88
00:25:47 [INFO] evaluate generation 503: reward = -89.02, steps = 91
00:25:49 [INFO] evaluate generation 504: reward = -90.75, steps = 88
00:25:51 [INFO] evaluate generation 505: reward = -90.20, steps = 86
00:25:53 [INFO] evaluate generation 506: reward = -90.14, steps = 86
00:25:54 [INFO] evaluate generation 507: reward = -88.79, steps = 91
00:25:57 [INFO] evaluate generation 508: reward = -89.01, steps = 93
00:25:59 [INFO] evaluate generation 509: reward = -88.18, steps = 97
00:26:02 [INFO] evaluate generation 510: reward = -87.74, steps = 99
00:26:04 [INFO] evaluate generation 511: reward = -87.66, steps = 102
00:26:10 [INFO] evaluate generation 512: reward = -89.81, steps = 89
00:26:12 [INFO] evaluate generation 513: reward = -90.73, steps = 86
00:26:14 [INFO] evaluate generation 514: reward = -90.34, steps = 87
00:26:16 [INFO] evaluate generation 515: reward = -90.02, steps = 87
00:26:18 [INFO] evaluate generation 516: reward = -92.04, steps = 83
00:26:19 [INFO] evaluate generation 517: reward = -89.00, steps = 96
00:26:22 [INFO] evaluate generation 518: reward = -88.77, steps = 97
00:26:24 [INFO] evaluate generation 519: reward = -89.15, steps = 94
00:26:26 [INFO] evaluate generation 520: reward = -87.48, steps = 101
00:26:28 [INFO] evaluate generation 521: reward = -88.00, steps = 103
00:26:30 [INFO] evaluate generation 522: reward = -87.68, steps = 105
00:26:32 [INFO] evaluate generation 523: reward = -86.58, steps = 107
00:26:35 [INFO] evaluate generation 524: reward = -86.04, steps = 108
00:26:38 [INFO] evaluate generation 525: reward = -88.93, steps = 94
00:26:40 [INFO] evaluate generation 526: reward = -88.36, steps = 96
00:26:42 [INFO] evaluate generation 527: reward = -88.80, steps = 102
00:26:44 [INFO] evaluate generation 528: reward = -87.51, steps = 103
00:26:46 [INFO] evaluate generation 529: reward = -86.62, steps = 105
00:26:48 [INFO] evaluate generation 530: reward = -86.29, steps = 110
00:26:51 [INFO] evaluate generation 531: reward = -87.47, steps = 99
00:26:54 [INFO] evaluate generation 532: reward = -87.80, steps = 99
00:26:56 [INFO] evaluate generation 533: reward = -88.22, steps = 96
00:26:58 [INFO] evaluate generation 534: reward = -88.29, steps = 100
00:27:00 [INFO] evaluate generation 535: reward = -88.20, steps = 94
00:27:02 [INFO] evaluate generation 536: reward = -87.14, steps = 107
00:27:04 [INFO] evaluate generation 537: reward = -86.95, steps = 104
00:27:06 [INFO] evaluate generation 538: reward = -86.34, steps = 103
00:27:08 [INFO] evaluate generation 539: reward = -87.32, steps = 103
00:27:10 [INFO] evaluate generation 540: reward = -85.30, steps = 109
00:27:14 [INFO] evaluate generation 541: reward = -83.47, steps = 115
00:27:16 [INFO] evaluate generation 542: reward = -84.33, steps = 107
00:27:18 [INFO] evaluate generation 543: reward = -82.28, steps = 115
00:27:21 [INFO] evaluate generation 544: reward = -81.42, steps = 120
00:27:24 [INFO] evaluate generation 545: reward = -81.54, steps = 121
00:27:26 [INFO] evaluate generation 546: reward = -82.79, steps = 112
00:27:29 [INFO] evaluate generation 547: reward = -80.56, steps = 125
00:27:31 [INFO] evaluate generation 548: reward = -68.97, steps = 170
00:27:35 [INFO] evaluate generation 549: reward = -65.92, steps = 197
00:27:39 [INFO] evaluate generation 550: reward = -81.30, steps = 114
00:27:41 [INFO] evaluate generation 551: reward = -81.11, steps = 118
00:27:43 [INFO] evaluate generation 552: reward = -83.24, steps = 107
00:27:45 [INFO] evaluate generation 553: reward = -72.81, steps = 132
00:27:47 [INFO] evaluate generation 554: reward = -101.91, steps = 135
00:27:50 [INFO] evaluate generation 555: reward = -80.77, steps = 114
00:27:52 [INFO] evaluate generation 556: reward = -82.26, steps = 116
00:27:54 [INFO] evaluate generation 557: reward = -82.73, steps = 116
00:27:56 [INFO] evaluate generation 558: reward = -82.38, steps = 116
00:27:58 [INFO] evaluate generation 559: reward = -78.81, steps = 126
00:28:01 [INFO] evaluate generation 560: reward = -84.63, steps = 98
00:28:04 [INFO] evaluate generation 561: reward = -73.29, steps = 166
00:28:07 [INFO] evaluate generation 562: reward = -76.76, steps = 135
00:28:09 [INFO] evaluate generation 563: reward = -67.43, steps = 192
00:28:12 [INFO] evaluate generation 564: reward = -71.69, steps = 155
00:28:15 [INFO] evaluate generation 565: reward = -83.45, steps = 118
00:28:18 [INFO] evaluate generation 566: reward = -63.22, steps = 211
00:28:21 [INFO] evaluate generation 567: reward = -67.10, steps = 233
00:28:25 [INFO] evaluate generation 568: reward = -66.44, steps = 191
00:28:28 [INFO] evaluate generation 569: reward = -64.18, steps = 195
00:28:32 [INFO] evaluate generation 570: reward = -57.64, steps = 212
00:28:36 [INFO] evaluate generation 571: reward = -53.65, steps = 283
00:28:43 [INFO] evaluate generation 572: reward = -75.04, steps = 135
00:28:45 [INFO] evaluate generation 573: reward = -69.18, steps = 155
00:28:48 [INFO] evaluate generation 574: reward = -75.50, steps = 136
00:28:53 [INFO] evaluate generation 575: reward = -80.18, steps = 389
00:29:02 [INFO] evaluate generation 576: reward = -86.52, steps = 104
00:29:06 [INFO] evaluate generation 577: reward = -75.12, steps = 268
00:29:20 [INFO] evaluate generation 578: reward = -88.71, steps = 111
00:29:24 [INFO] evaluate generation 579: reward = -3.23, steps = 721
00:29:32 [INFO] evaluate generation 580: reward = -53.31, steps = 234
00:29:39 [INFO] evaluate generation 581: reward = -53.73, steps = 310
00:29:45 [INFO] evaluate generation 582: reward = 40.25, steps = 1378
00:30:00 [INFO] evaluate generation 583: reward = -95.35, steps = 121
00:30:11 [INFO] evaluate generation 584: reward = -107.11, steps = 62
00:30:13 [INFO] evaluate generation 585: reward = -103.34, steps = 55
00:30:15 [INFO] evaluate generation 586: reward = -102.47, steps = 57
00:30:16 [INFO] evaluate generation 587: reward = -96.87, steps = 87
00:30:18 [INFO] evaluate generation 588: reward = -98.55, steps = 72
00:30:20 [INFO] evaluate generation 589: reward = -98.40, steps = 73
00:30:22 [INFO] evaluate generation 590: reward = -93.94, steps = 88
00:30:24 [INFO] evaluate generation 591: reward = -75.77, steps = 164
00:30:26 [INFO] evaluate generation 592: reward = -80.02, steps = 158
00:30:28 [INFO] evaluate generation 593: reward = -73.42, steps = 200
00:30:33 [INFO] evaluate generation 594: reward = -80.04, steps = 135
00:30:43 [INFO] evaluate generation 595: reward = 60.82, steps = 1600
00:31:00 [INFO] evaluate generation 596: reward = -97.49, steps = 83
00:31:07 [INFO] evaluate generation 597: reward = -128.73, steps = 1600
00:31:22 [INFO] evaluate generation 598: reward = -98.18, steps = 109
00:31:29 [INFO] evaluate generation 599: reward = -94.99, steps = 94
00:31:30 [INFO] evaluate generation 600: reward = -94.69, steps = 90
00:31:32 [INFO] evaluate generation 601: reward = -93.46, steps = 100
00:31:35 [INFO] evaluate generation 602: reward = -94.29, steps = 98
00:31:37 [INFO] evaluate generation 603: reward = -93.04, steps = 103
00:31:39 [INFO] evaluate generation 604: reward = -94.55, steps = 104
00:31:41 [INFO] evaluate generation 605: reward = -91.30, steps = 106
00:31:44 [INFO] evaluate generation 606: reward = -95.63, steps = 94
00:31:46 [INFO] evaluate generation 607: reward = -90.81, steps = 108
00:31:48 [INFO] evaluate generation 608: reward = -92.17, steps = 103
00:31:51 [INFO] evaluate generation 609: reward = -91.89, steps = 106
00:31:53 [INFO] evaluate generation 610: reward = -92.40, steps = 104
00:31:55 [INFO] evaluate generation 611: reward = -91.65, steps = 102
00:31:58 [INFO] evaluate generation 612: reward = -94.01, steps = 95
00:32:00 [INFO] evaluate generation 613: reward = -90.92, steps = 106
00:32:02 [INFO] evaluate generation 614: reward = -92.46, steps = 96
00:32:04 [INFO] evaluate generation 615: reward = -91.80, steps = 107
00:32:07 [INFO] evaluate generation 616: reward = -88.82, steps = 107
00:32:09 [INFO] evaluate generation 617: reward = -90.34, steps = 107
00:32:12 [INFO] evaluate generation 618: reward = -91.84, steps = 105
00:32:14 [INFO] evaluate generation 619: reward = -87.39, steps = 124
00:32:18 [INFO] evaluate generation 620: reward = -88.07, steps = 124
00:32:22 [INFO] evaluate generation 621: reward = -87.00, steps = 111
00:32:29 [INFO] evaluate generation 622: reward = -83.72, steps = 126
00:32:41 [INFO] evaluate generation 623: reward = 30.28, steps = 1600
00:33:01 [INFO] evaluate generation 624: reward = -115.19, steps = 83
00:33:03 [INFO] evaluate generation 625: reward = -64.10, steps = 323
00:33:13 [INFO] evaluate generation 626: reward = -78.63, steps = 527
00:33:29 [INFO] evaluate generation 627: reward = -45.67, steps = 609
00:33:37 [INFO] evaluate generation 628: reward = -8.58, steps = 1600
00:34:01 [INFO] evaluate generation 629: reward = -97.06, steps = 93
00:34:03 [INFO] evaluate generation 630: reward = -88.63, steps = 144
00:34:06 [INFO] evaluate generation 631: reward = -56.30, steps = 340
00:34:12 [INFO] evaluate generation 632: reward = 53.08, steps = 1600
00:34:40 [INFO] evaluate generation 633: reward = -98.89, steps = 68
00:34:41 [INFO] evaluate generation 634: reward = -94.84, steps = 81
00:34:43 [INFO] evaluate generation 635: reward = -94.63, steps = 83
00:34:46 [INFO] evaluate generation 636: reward = -79.27, steps = 154
00:34:52 [INFO] evaluate generation 637: reward = -71.75, steps = 183
00:35:12 [INFO] evaluate generation 638: reward = -93.78, steps = 93
00:35:18 [INFO] evaluate generation 639: reward = -15.95, steps = 857
00:35:34 [INFO] evaluate generation 640: reward = 65.04, steps = 1600
00:35:59 [INFO] evaluate generation 641: reward = -97.11, steps = 80
00:36:04 [INFO] evaluate generation 642: reward = 135.59, steps = 1600
00:36:28 [INFO] evaluate generation 643: reward = -115.13, steps = 70
00:36:30 [INFO] evaluate generation 644: reward = -111.53, steps = 71
00:36:32 [INFO] evaluate generation 645: reward = -77.57, steps = 169
00:36:36 [INFO] evaluate generation 646: reward = -71.62, steps = 402
00:36:44 [INFO] evaluate generation 647: reward = -62.37, steps = 399
00:37:01 [INFO] evaluate generation 648: reward = -68.73, steps = 307
00:37:08 [INFO] evaluate generation 649: reward = -77.86, steps = 1600
00:37:35 [INFO] evaluate generation 650: reward = -47.39, steps = 439
00:37:46 [INFO] evaluate generation 651: reward = 20.01, steps = 1100
00:37:59 [INFO] evaluate generation 652: reward = 56.53, steps = 1600
00:38:29 [INFO] evaluate generation 653: reward = 69.16, steps = 1600
00:38:58 [INFO] evaluate generation 654: reward = 98.09, steps = 1600
00:39:24 [INFO] evaluate generation 655: reward = 45.66, steps = 1600
00:39:54 [INFO] evaluate generation 656: reward = -29.98, steps = 795
00:40:13 [INFO] evaluate generation 657: reward = -38.59, steps = 1600
00:40:45 [INFO] evaluate generation 658: reward = 1.47, steps = 1600
00:41:15 [INFO] evaluate generation 659: reward = 114.46, steps = 1600
00:41:44 [INFO] evaluate generation 660: reward = 24.30, steps = 1026
00:42:02 [INFO] evaluate generation 661: reward = 25.24, steps = 1600
00:42:33 [INFO] evaluate generation 662: reward = 26.67, steps = 1600
00:43:02 [INFO] evaluate generation 663: reward = 197.75, steps = 1600
00:43:27 [INFO] evaluate generation 664: reward = 93.20, steps = 1600
00:43:57 [INFO] evaluate generation 665: reward = 168.31, steps = 1600
00:44:22 [INFO] evaluate generation 666: reward = 127.65, steps = 1600
00:44:51 [INFO] evaluate generation 667: reward = -70.96, steps = 317
00:45:07 [INFO] evaluate generation 668: reward = -25.58, steps = 1600
00:45:38 [INFO] evaluate generation 669: reward = -9.13, steps = 1600
00:46:08 [INFO] evaluate generation 670: reward = 63.14, steps = 1600
00:46:38 [INFO] evaluate generation 671: reward = 10.36, steps = 1600
00:47:06 [INFO] evaluate generation 672: reward = -121.13, steps = 98
00:47:13 [INFO] evaluate generation 673: reward = 83.24, steps = 1600
00:47:38 [INFO] evaluate generation 674: reward = -42.57, steps = 1577
00:48:04 [INFO] evaluate generation 675: reward = 71.36, steps = 1600
00:48:28 [INFO] evaluate generation 676: reward = -113.17, steps = 777
00:48:53 [INFO] evaluate generation 677: reward = -64.53, steps = 654
00:49:16 [INFO] evaluate generation 678: reward = 56.95, steps = 1600
00:49:43 [INFO] evaluate generation 679: reward = -97.53, steps = 1600
00:50:06 [INFO] evaluate generation 680: reward = -108.92, steps = 69
00:50:20 [INFO] evaluate generation 681: reward = 78.01, steps = 1600
00:50:34 [INFO] evaluate generation 682: reward = -115.59, steps = 62
00:50:36 [INFO] evaluate generation 683: reward = -109.30, steps = 70
00:50:44 [INFO] evaluate generation 684: reward = -76.15, steps = 239
00:50:53 [INFO] evaluate generation 685: reward = -84.03, steps = 330
00:51:17 [INFO] evaluate generation 686: reward = -118.13, steps = 199
00:51:20 [INFO] evaluate generation 687: reward = -99.18, steps = 166
00:51:22 [INFO] evaluate generation 688: reward = -89.39, steps = 173
00:51:25 [INFO] evaluate generation 689: reward = -114.47, steps = 100
00:51:29 [INFO] evaluate generation 690: reward = -66.70, steps = 370
00:51:32 [INFO] evaluate generation 691: reward = -110.66, steps = 98
00:51:35 [INFO] evaluate generation 692: reward = -84.90, steps = 169
00:51:39 [INFO] evaluate generation 693: reward = -89.77, steps = 339
00:51:42 [INFO] evaluate generation 694: reward = -106.46, steps = 134
00:51:46 [INFO] evaluate generation 695: reward = -91.23, steps = 196
00:51:49 [INFO] evaluate generation 696: reward = -107.07, steps = 106
00:51:55 [INFO] evaluate generation 697: reward = -82.24, steps = 124
00:52:01 [INFO] evaluate generation 698: reward = -64.66, steps = 316
00:52:06 [INFO] evaluate generation 699: reward = -91.69, steps = 115
00:52:12 [INFO] evaluate generation 700: reward = -33.37, steps = 472
00:52:17 [INFO] evaluate generation 701: reward = -94.24, steps = 108
00:52:23 [INFO] evaluate generation 702: reward = -58.26, steps = 747
00:52:32 [INFO] evaluate generation 703: reward = -26.80, steps = 801
00:52:53 [INFO] evaluate generation 704: reward = -57.74, steps = 1494
00:53:21 [INFO] evaluate generation 705: reward = 158.33, steps = 1600
00:53:38 [INFO] evaluate generation 706: reward = -97.25, steps = 132
00:53:48 [INFO] evaluate generation 707: reward = -125.16, steps = 121
00:53:56 [INFO] evaluate generation 708: reward = 42.08, steps = 1438
00:54:09 [INFO] evaluate generation 709: reward = 115.54, steps = 1600
00:54:39 [INFO] evaluate generation 710: reward = -24.86, steps = 630
00:54:49 [INFO] evaluate generation 711: reward = 126.28, steps = 1600
00:55:17 [INFO] evaluate generation 712: reward = -32.66, steps = 500
00:55:38 [INFO] evaluate generation 713: reward = 87.51, steps = 1600
00:56:02 [INFO] evaluate generation 714: reward = -62.11, steps = 496
00:56:26 [INFO] evaluate generation 715: reward = 166.12, steps = 1600
00:56:55 [INFO] evaluate generation 716: reward = 126.81, steps = 1600
00:57:25 [INFO] evaluate generation 717: reward = 183.11, steps = 1600
00:57:53 [INFO] evaluate generation 718: reward = 120.87, steps = 1600
00:58:23 [INFO] evaluate generation 719: reward = 97.14, steps = 1600
00:58:54 [INFO] evaluate generation 720: reward = 77.86, steps = 1600
00:59:22 [INFO] evaluate generation 721: reward = -72.93, steps = 429
00:59:39 [INFO] evaluate generation 722: reward = 176.06, steps = 1600
01:00:04 [INFO] evaluate generation 723: reward = -121.92, steps = 1600
01:00:33 [INFO] evaluate generation 724: reward = 121.38, steps = 1600
01:01:03 [INFO] evaluate generation 725: reward = -97.63, steps = 90
01:01:07 [INFO] evaluate generation 726: reward = -102.87, steps = 133
01:01:17 [INFO] evaluate generation 727: reward = -30.24, steps = 710
01:01:40 [INFO] evaluate generation 728: reward = -95.48, steps = 107
01:01:43 [INFO] evaluate generation 729: reward = -90.10, steps = 160
01:01:48 [INFO] evaluate generation 730: reward = -84.08, steps = 201
01:01:55 [INFO] evaluate generation 731: reward = -53.14, steps = 335
01:02:12 [INFO] evaluate generation 732: reward = 122.44, steps = 1600
01:02:42 [INFO] evaluate generation 733: reward = -139.17, steps = 1600
01:03:06 [INFO] evaluate generation 734: reward = -99.76, steps = 97
01:03:28 [INFO] evaluate generation 735: reward = 110.30, steps = 1600
01:03:56 [INFO] evaluate generation 736: reward = 104.57, steps = 1600
01:04:23 [INFO] evaluate generation 737: reward = 92.69, steps = 1600
01:04:49 [INFO] evaluate generation 738: reward = 102.10, steps = 1600
01:05:18 [INFO] evaluate generation 739: reward = 121.05, steps = 1600
01:05:47 [INFO] evaluate generation 740: reward = 97.34, steps = 1600
01:06:17 [INFO] evaluate generation 741: reward = -27.50, steps = 966
01:06:48 [INFO] evaluate generation 742: reward = -77.51, steps = 579
01:07:16 [INFO] evaluate generation 743: reward = -16.90, steps = 1019
01:07:43 [INFO] evaluate generation 744: reward = -125.51, steps = 1600
01:08:10 [INFO] evaluate generation 745: reward = 136.72, steps = 1600
01:08:33 [INFO] evaluate generation 746: reward = -81.06, steps = 346
01:08:58 [INFO] evaluate generation 747: reward = 143.35, steps = 1600
01:09:28 [INFO] evaluate generation 748: reward = 129.72, steps = 1600
01:09:55 [INFO] evaluate generation 749: reward = 86.71, steps = 1600
01:10:25 [INFO] evaluate generation 750: reward = 114.81, steps = 1600
01:10:54 [INFO] evaluate generation 751: reward = 117.70, steps = 1600
01:11:23 [INFO] evaluate generation 752: reward = 140.29, steps = 1600
01:11:52 [INFO] evaluate generation 753: reward = 141.92, steps = 1600
01:12:21 [INFO] evaluate generation 754: reward = 117.98, steps = 1600
01:12:50 [INFO] evaluate generation 755: reward = 122.68, steps = 1600
01:13:17 [INFO] evaluate generation 756: reward = 164.29, steps = 1600
01:13:45 [INFO] evaluate generation 757: reward = 170.40, steps = 1600
01:14:14 [INFO] evaluate generation 758: reward = -89.30, steps = 327
01:14:38 [INFO] evaluate generation 759: reward = 118.54, steps = 1600
01:15:07 [INFO] evaluate generation 760: reward = 161.18, steps = 1600
01:15:35 [INFO] evaluate generation 761: reward = 30.81, steps = 1001
01:15:53 [INFO] evaluate generation 762: reward = 47.21, steps = 1600
01:16:23 [INFO] evaluate generation 763: reward = 59.25, steps = 1600
01:16:53 [INFO] evaluate generation 764: reward = 95.86, steps = 1600
01:17:22 [INFO] evaluate generation 765: reward = 157.91, steps = 1600
01:17:53 [INFO] evaluate generation 766: reward = 150.65, steps = 1600
01:18:24 [INFO] evaluate generation 767: reward = 129.11, steps = 1600
01:18:54 [INFO] evaluate generation 768: reward = 124.15, steps = 1600
01:19:26 [INFO] evaluate generation 769: reward = 185.81, steps = 1600
01:19:57 [INFO] evaluate generation 770: reward = 141.42, steps = 1600
01:20:27 [INFO] evaluate generation 771: reward = 4.72, steps = 1097
01:20:59 [INFO] evaluate generation 772: reward = 204.74, steps = 1600
01:21:26 [INFO] evaluate generation 773: reward = -92.62, steps = 149
01:21:54 [INFO] evaluate generation 774: reward = 149.02, steps = 1600
01:22:25 [INFO] evaluate generation 775: reward = 190.06, steps = 1600
01:22:51 [INFO] evaluate generation 776: reward = 144.96, steps = 1600
01:23:20 [INFO] evaluate generation 777: reward = 170.60, steps = 1600
01:23:51 [INFO] evaluate generation 778: reward = 150.15, steps = 1600
01:24:21 [INFO] evaluate generation 779: reward = 164.23, steps = 1600
01:24:50 [INFO] evaluate generation 780: reward = 158.35, steps = 1600
01:25:19 [INFO] evaluate generation 781: reward = -51.99, steps = 591
01:25:45 [INFO] evaluate generation 782: reward = 190.31, steps = 1600
01:26:10 [INFO] evaluate generation 783: reward = 17.73, steps = 1276
01:26:36 [INFO] evaluate generation 784: reward = 219.54, steps = 1600
01:27:04 [INFO] evaluate generation 785: reward = 192.08, steps = 1600
01:27:32 [INFO] evaluate generation 786: reward = 197.40, steps = 1600
01:27:56 [INFO] evaluate generation 787: reward = 98.78, steps = 1600
01:28:26 [INFO] evaluate generation 788: reward = 108.58, steps = 1600
01:28:57 [INFO] evaluate generation 789: reward = 131.42, steps = 1600
01:29:28 [INFO] evaluate generation 790: reward = 206.52, steps = 1600
01:29:58 [INFO] evaluate generation 791: reward = 200.44, steps = 1600
01:30:31 [INFO] evaluate generation 792: reward = 196.37, steps = 1600
01:30:59 [INFO] evaluate generation 793: reward = -49.54, steps = 1055
01:31:25 [INFO] evaluate generation 794: reward = -13.57, steps = 734
01:31:54 [INFO] evaluate generation 795: reward = 223.32, steps = 1556
01:32:16 [INFO] evaluate generation 796: reward = 183.09, steps = 1600
01:32:45 [INFO] evaluate generation 797: reward = 181.30, steps = 1600
01:33:12 [INFO] evaluate generation 798: reward = 164.70, steps = 1600
01:33:43 [INFO] evaluate generation 799: reward = 194.39, steps = 1600
01:34:11 [INFO] evaluate generation 800: reward = 204.27, steps = 1600
01:34:42 [INFO] evaluate generation 801: reward = 203.59, steps = 1600
01:35:14 [INFO] evaluate generation 802: reward = 208.83, steps = 1600
01:35:41 [INFO] evaluate generation 803: reward = -18.96, steps = 625
01:36:05 [INFO] evaluate generation 804: reward = 160.46, steps = 1600
01:36:35 [INFO] evaluate generation 805: reward = 208.16, steps = 1600
01:37:05 [INFO] evaluate generation 806: reward = 190.17, steps = 1600
01:37:34 [INFO] evaluate generation 807: reward = 170.70, steps = 1600
01:38:05 [INFO] evaluate generation 808: reward = 144.47, steps = 1600
01:38:35 [INFO] evaluate generation 809: reward = 219.74, steps = 1583
01:39:04 [INFO] evaluate generation 810: reward = 215.70, steps = 1581
01:39:27 [INFO] evaluate generation 811: reward = 39.29, steps = 1403
01:39:53 [INFO] evaluate generation 812: reward = 197.13, steps = 1600
01:40:20 [INFO] evaluate generation 813: reward = 177.28, steps = 1600
01:40:48 [INFO] evaluate generation 814: reward = 209.48, steps = 1600
01:41:15 [INFO] evaluate generation 815: reward = 18.35, steps = 1028
01:41:43 [INFO] evaluate generation 816: reward = 227.63, steps = 1461
01:42:06 [INFO] evaluate generation 817: reward = 217.51, steps = 1574
01:42:34 [INFO] evaluate generation 818: reward = 199.50, steps = 1600
01:43:06 [INFO] evaluate generation 819: reward = 215.12, steps = 1597
01:43:32 [INFO] evaluate generation 820: reward = 208.63, steps = 1600
01:44:02 [INFO] evaluate generation 821: reward = 224.07, steps = 1486
01:44:30 [INFO] evaluate generation 822: reward = 210.61, steps = 1600
01:44:57 [INFO] evaluate generation 823: reward = 220.62, steps = 1514
01:45:25 [INFO] evaluate generation 824: reward = 194.16, steps = 1600
01:45:56 [INFO] evaluate generation 825: reward = 204.81, steps = 1600
01:45:56 [INFO] ==== test ====
01:45:57 [INFO] test episode 0: reward = 198.63, steps = 1600
01:45:57 [INFO] test episode 1: reward = 214.30, steps = 1600
01:45:58 [INFO] test episode 2: reward = 187.62, steps = 1600
01:45:59 [INFO] test episode 3: reward = 188.54, steps = 1600
01:46:00 [INFO] test episode 4: reward = 201.10, steps = 1600
01:46:01 [INFO] test episode 5: reward = 191.86, steps = 1600
01:46:02 [INFO] test episode 6: reward = 169.99, steps = 1600
01:46:03 [INFO] test episode 7: reward = 219.32, steps = 1549
01:46:04 [INFO] test episode 8: reward = 206.25, steps = 1600
01:46:05 [INFO] test episode 9: reward = 181.40, steps = 1600
01:46:05 [INFO] test episode 10: reward = 159.98, steps = 1600
01:46:06 [INFO] test episode 11: reward = 185.03, steps = 1600
01:46:07 [INFO] test episode 12: reward = 208.14, steps = 1600
01:46:08 [INFO] test episode 13: reward = 199.01, steps = 1600
01:46:09 [INFO] test episode 14: reward = 142.58, steps = 1600
01:46:10 [INFO] test episode 15: reward = 157.67, steps = 1600
01:46:11 [INFO] test episode 16: reward = 178.38, steps = 1600
01:46:12 [INFO] test episode 17: reward = 184.99, steps = 1600
01:46:13 [INFO] test episode 18: reward = 196.74, steps = 1600
01:46:14 [INFO] test episode 19: reward = 183.32, steps = 1600
01:46:15 [INFO] test episode 20: reward = 209.74, steps = 1600
01:46:16 [INFO] test episode 21: reward = 164.58, steps = 1600
01:46:16 [INFO] test episode 22: reward = 192.50, steps = 1600
01:46:17 [INFO] test episode 23: reward = 24.11, steps = 1542
01:46:18 [INFO] test episode 24: reward = 170.73, steps = 1600
01:46:19 [INFO] test episode 25: reward = 198.98, steps = 1600
01:46:20 [INFO] test episode 26: reward = 186.42, steps = 1600
01:46:21 [INFO] test episode 27: reward = 205.76, steps = 1600
01:46:22 [INFO] test episode 28: reward = 205.77, steps = 1600
01:46:23 [INFO] test episode 29: reward = 200.57, steps = 1600
01:46:24 [INFO] test episode 30: reward = 194.09, steps = 1600
01:46:24 [INFO] test episode 31: reward = 200.73, steps = 1600
01:46:25 [INFO] test episode 32: reward = 213.55, steps = 1572
01:46:26 [INFO] test episode 33: reward = 202.87, steps = 1600
01:46:27 [INFO] test episode 34: reward = 210.49, steps = 1600
01:46:28 [INFO] test episode 35: reward = 53.06, steps = 1543
01:46:29 [INFO] test episode 36: reward = 174.22, steps = 1600
01:46:30 [INFO] test episode 37: reward = 162.57, steps = 1600
01:46:31 [INFO] test episode 38: reward = 183.49, steps = 1600
01:46:31 [INFO] test episode 39: reward = 206.19, steps = 1600
01:46:32 [INFO] test episode 40: reward = 215.01, steps = 1591
01:46:33 [INFO] test episode 41: reward = 189.69, steps = 1600
01:46:34 [INFO] test episode 42: reward = 212.72, steps = 1600
01:46:35 [INFO] test episode 43: reward = 203.49, steps = 1600
01:46:36 [INFO] test episode 44: reward = 178.35, steps = 1600
01:46:36 [INFO] test episode 45: reward = 216.92, steps = 1546
01:46:37 [INFO] test episode 46: reward = 179.52, steps = 1600
01:46:38 [INFO] test episode 47: reward = 221.49, steps = 1514
01:46:39 [INFO] test episode 48: reward = 201.52, steps = 1600
01:46:40 [INFO] test episode 49: reward = 194.13, steps = 1600
01:46:41 [INFO] test episode 50: reward = 214.08, steps = 1598
01:46:42 [INFO] test episode 51: reward = 201.02, steps = 1600
01:46:42 [INFO] test episode 52: reward = 209.53, steps = 1600
01:46:43 [INFO] test episode 53: reward = 189.02, steps = 1600
01:46:44 [INFO] test episode 54: reward = 218.71, steps = 1526
01:46:45 [INFO] test episode 55: reward = 206.27, steps = 1600
01:46:46 [INFO] test episode 56: reward = 210.93, steps = 1599
01:46:47 [INFO] test episode 57: reward = 217.93, steps = 1563
01:46:48 [INFO] test episode 58: reward = 209.36, steps = 1600
01:46:48 [INFO] test episode 59: reward = 220.93, steps = 1529
01:46:49 [INFO] test episode 60: reward = 220.72, steps = 1530
01:46:50 [INFO] test episode 61: reward = 207.05, steps = 1600
01:46:51 [INFO] test episode 62: reward = 179.34, steps = 1600
01:46:52 [INFO] test episode 63: reward = 200.20, steps = 1600
01:46:53 [INFO] test episode 64: reward = 190.57, steps = 1600
01:46:54 [INFO] test episode 65: reward = 184.90, steps = 1600
01:46:54 [INFO] test episode 66: reward = 220.28, steps = 1517
01:46:55 [INFO] test episode 67: reward = 172.18, steps = 1600
01:46:56 [INFO] test episode 68: reward = 214.25, steps = 1584
01:46:57 [INFO] test episode 69: reward = 225.22, steps = 1458
01:46:58 [INFO] test episode 70: reward = 214.61, steps = 1585
01:46:58 [INFO] test episode 71: reward = 198.05, steps = 1600
01:46:59 [INFO] test episode 72: reward = 205.03, steps = 1600
01:47:00 [INFO] test episode 73: reward = 197.68, steps = 1600
01:47:01 [INFO] test episode 74: reward = 159.38, steps = 1600
01:47:02 [INFO] test episode 75: reward = 195.35, steps = 1600
01:47:03 [INFO] test episode 76: reward = 217.50, steps = 1552
01:47:04 [INFO] test episode 77: reward = 181.22, steps = 1600
01:47:04 [INFO] test episode 78: reward = 202.95, steps = 1600
01:47:05 [INFO] test episode 79: reward = 202.03, steps = 1600
01:47:06 [INFO] test episode 80: reward = 186.07, steps = 1600
01:47:07 [INFO] test episode 81: reward = 219.71, steps = 1536
01:47:08 [INFO] test episode 82: reward = 74.13, steps = 1436
01:47:09 [INFO] test episode 83: reward = 209.32, steps = 1600
01:47:09 [INFO] test episode 84: reward = 208.08, steps = 1600
01:47:10 [INFO] test episode 85: reward = 193.19, steps = 1600
01:47:11 [INFO] test episode 86: reward = 164.39, steps = 1600
01:47:12 [INFO] test episode 87: reward = 202.00, steps = 1600
01:47:13 [INFO] test episode 88: reward = 202.55, steps = 1600
01:47:14 [INFO] test episode 89: reward = 225.22, steps = 1466
01:47:15 [INFO] test episode 90: reward = 161.39, steps = 1600
01:47:15 [INFO] test episode 91: reward = 193.27, steps = 1600
01:47:16 [INFO] test episode 92: reward = 159.64, steps = 1600
01:47:17 [INFO] test episode 93: reward = 200.44, steps = 1600
01:47:18 [INFO] test episode 94: reward = 170.55, steps = 1600
01:47:19 [INFO] test episode 95: reward = 216.36, steps = 1568
01:47:20 [INFO] test episode 96: reward = 215.38, steps = 1581
01:47:21 [INFO] test episode 97: reward = 213.69, steps = 1594
01:47:22 [INFO] test episode 98: reward = 202.27, steps = 1600
01:47:22 [INFO] test episode 99: reward = 164.51, steps = 1600
01:47:22 [INFO] average episode reward = 191.97 ± 30.95