Use Soft Actor-Critic to Play LunarLander-v2

PyTorch version

%matplotlib inline

import sys
import logging
import itertools
import copy

import numpy as np
np.random.seed(0)
import pandas as pd
import gym
import matplotlib.pyplot as plt
import torch
torch.manual_seed(0)
import torch.nn as nn
import torch.optim as optim
import torch.distributions as distributions

logging.basicConfig(level=logging.INFO,
        format='%(asctime)s [%(levelname)s] %(message)s',
        stream=sys.stdout, datefmt='%H:%M:%S')

env = gym.make('LunarLander-v2')
for key in vars(env):
    logging.info('%s: %s', key, vars(env)[key])
for key in vars(env.spec):
    logging.info('%s: %s', key, vars(env.spec)[key])

06:44:32 [INFO] env: <LunarLander<LunarLander-v2>>
06:44:32 [INFO] action_space: Discrete(4)
06:44:32 [INFO] observation_space: Box(-inf, inf, (8,), float32)
06:44:32 [INFO] reward_range: (-inf, inf)
06:44:32 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}
06:44:32 [INFO] _max_episode_steps: 1000
06:44:32 [INFO] _elapsed_steps: None
06:44:32 [INFO] id: LunarLander-v2
06:44:32 [INFO] entry_point: gym.envs.box2d:LunarLander
06:44:32 [INFO] reward_threshold: 200
06:44:32 [INFO] nondeterministic: False
06:44:32 [INFO] max_episode_steps: 1000
06:44:32 [INFO] _kwargs: {}
06:44:32 [INFO] _env_name: LunarLander

class DQNReplayer:
    def __init__(self, capacity):
        self.memory = pd.DataFrame(index=range(capacity),
                columns=['state', 'action', 'reward', 'next_state', 'terminated'])
        self.i = 0
        self.count = 0
        self.capacity = capacity

    def store(self, *args):
        self.memory.loc[self.i] = np.asarray(args, dtype=object)
        self.i = (self.i + 1) % self.capacity
        self.count = min(self.count + 1, self.capacity)

    def sample(self, size):
        indices = np.random.choice(self.count, size=size)
        return (np.stack(self.memory.loc[indices, field]) for field in
                self.memory.columns)

class SACAgent:
    def __init__(self, env):
        state_dim = env.observation_space.shape[0]
        self.action_n = env.action_space.n
        self.gamma = 0.99
        self.replayer = DQNReplayer(10000)

        self.alpha = 0.02

        # create actor
        self.actor_net = self.build_net(input_size=state_dim,
                hidden_sizes=[256, 256],
                output_size=self.action_n, output_activator=nn.Softmax(-1))
        self.actor_optimizer = optim.Adam(self.actor_net.parameters(), lr=3e-4)

        # create V critic
        self.v_evaluate_net = self.build_net(input_size=state_dim,
                hidden_sizes=[256, 256])
        self.v_target_net = copy.deepcopy(self.v_evaluate_net)
        self.v_optimizer = optim.Adam(self.v_evaluate_net.parameters(), lr=3e-4)
        self.v_loss = nn.MSELoss()

        # create Q critic
        self.q0_net = self.build_net(input_size=state_dim,
                hidden_sizes=[256, 256], output_size=self.action_n)
        self.q1_net = self.build_net(input_size=state_dim,
                hidden_sizes=[256, 256], output_size=self.action_n)
        self.q0_loss = nn.MSELoss()
        self.q1_loss = nn.MSELoss()
        self.q0_optimizer = optim.Adam(self.q0_net.parameters(), lr=3e-4)
        self.q1_optimizer = optim.Adam(self.q1_net.parameters(), lr=3e-4)

    def build_net(self, input_size, hidden_sizes, output_size=1,
            output_activator=None):
        layers = []
        for input_size, output_size in zip(
                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):
            layers.append(nn.Linear(input_size, output_size))
            layers.append(nn.ReLU())
        layers = layers[:-1]
        if output_activator:
            layers.append(output_activator)
        net = nn.Sequential(*layers)
        return net

    def reset(self, mode=None):
        self.mode = mode
        if self.mode == 'train':
            self.trajectory = []

    def step(self, observation, reward, terminated):
        state_tensor = torch.as_tensor(observation, dtype=torch.float).unsqueeze(0)
        prob_tensor = self.actor_net(state_tensor)
        action_tensor = distributions.Categorical(prob_tensor).sample()
        action = action_tensor.numpy()[0]
        if self.mode == 'train':
            self.trajectory += [observation, reward, terminated, action]
            if len(self.trajectory) >= 8:
                state, _, _, action, next_state, reward, terminated, _ = \
                        self.trajectory[-8:]
                self.replayer.store(state, action, reward, next_state, terminated)
            if self.replayer.count >= 500:
                self.learn()
        return action

    def close(self):
        pass

    def update_net(self, target_net, evaluate_net, learning_rate=0.0025):
        for target_param, evaluate_param in zip(
                target_net.parameters(), evaluate_net.parameters()):
            target_param.data.copy_(learning_rate * evaluate_param.data
                    + (1 - learning_rate) * target_param.data)

    def learn(self):
        states, actions, rewards, next_states, terminateds = \
                self.replayer.sample(128)
        state_tensor = torch.as_tensor(states, dtype=torch.float)
        action_tensor = torch.as_tensor(actions, dtype=torch.long)
        reward_tensor = torch.as_tensor(rewards, dtype=torch.float)
        next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)
        terminated_tensor = torch.as_tensor(terminateds, dtype=torch.float)

        # update Q critic
        next_v_tensor = self.v_target_net(next_state_tensor)
        q_target_tensor = reward_tensor.unsqueeze(1) + self.gamma * \
                (1. - terminated_tensor.unsqueeze(1)) * next_v_tensor

        all_q0_pred_tensor = self.q0_net(state_tensor)
        q0_pred_tensor = torch.gather(all_q0_pred_tensor, 1,
                action_tensor.unsqueeze(1))
        q0_loss_tensor = self.q0_loss(q0_pred_tensor, q_target_tensor.detach())
        self.q0_optimizer.zero_grad()
        q0_loss_tensor.backward()
        self.q0_optimizer.step()

        all_q1_pred_tensor = self.q1_net(state_tensor)
        q1_pred_tensor = torch.gather(all_q1_pred_tensor, 1,
                action_tensor.unsqueeze(1))
        q1_loss_tensor = self.q1_loss(q1_pred_tensor, q_target_tensor.detach())
        self.q1_optimizer.zero_grad()
        q1_loss_tensor.backward()
        self.q1_optimizer.step()

        # update V critic
        q0_tensor = self.q0_net(state_tensor)
        q1_tensor = self.q1_net(state_tensor)
        q01_tensor = torch.min(q0_tensor, q1_tensor)
        prob_tensor = self.actor_net(state_tensor)
        ln_prob_tensor = torch.log(prob_tensor.clamp(1e-6, 1.))
        entropic_q01_tensor = prob_tensor * (q01_tensor -
                self.alpha * ln_prob_tensor)
        # OR entropic_q01_tensor = prob_tensor * (q01_tensor - \
        #         self.alpha * torch.xlogy(prob_tensor, prob_tensor)
        v_target_tensor = torch.sum(entropic_q01_tensor, dim=-1, keepdim=True)
        v_pred_tensor = self.v_evaluate_net(state_tensor)
        v_loss_tensor = self.v_loss(v_pred_tensor, v_target_tensor.detach())
        self.v_optimizer.zero_grad()
        v_loss_tensor.backward()
        self.v_optimizer.step()

        self.update_net(self.v_target_net, self.v_evaluate_net)

        # update actor
        prob_q_tensor = prob_tensor * (self.alpha * ln_prob_tensor - q0_tensor)
        actor_loss_tensor = prob_q_tensor.sum(axis=-1).mean()
        self.actor_optimizer.zero_grad()
        actor_loss_tensor.backward()
        self.actor_optimizer.step()


agent = SACAgent(env)

def play_episode(env, agent, seed=None, mode=None, render=False):
    observation, _ = env.reset(seed=seed)
    reward, terminated, truncated = 0., False, False
    agent.reset(mode=mode)
    episode_reward, elapsed_steps = 0., 0
    while True:
        action = agent.step(observation, reward, terminated)
        if render:
            env.render()
        if terminated or truncated:
            break
        observation, reward, terminated, truncated, _ = env.step(action)
        episode_reward += reward
        elapsed_steps += 1
    agent.close()
    return episode_reward, elapsed_steps


logging.info('==== train ====')
episode_rewards = []
for episode in itertools.count():
    episode_reward, elapsed_steps = play_episode(env, agent, seed=episode,
            mode='train')
    episode_rewards.append(episode_reward)
    logging.info('train episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
    if np.mean(episode_rewards[-10:]) > 200:
        break
plt.plot(episode_rewards)


logging.info('==== test ====')
episode_rewards = []
for episode in range(100):
    episode_reward, elapsed_steps = play_episode(env, agent)
    episode_rewards.append(episode_reward)
    logging.info('test episode %d: reward = %.2f, steps = %d',
            episode, episode_reward, elapsed_steps)
logging.info('average episode reward = %.2f ± %.2f',
        np.mean(episode_rewards), np.std(episode_rewards))

06:44:33 [INFO] ==== train ====
06:44:33 [INFO] train episode 0: reward = -119.74, steps = 100
06:44:33 [INFO] train episode 1: reward = -53.41, steps = 63
06:44:33 [INFO] train episode 2: reward = -136.45, steps = 87
06:44:33 [INFO] train episode 3: reward = -326.87, steps = 70
06:44:33 [INFO] train episode 4: reward = -117.95, steps = 113
06:44:34 [INFO] train episode 5: reward = -431.55, steps = 99
06:44:37 [INFO] train episode 6: reward = -344.16, steps = 79
06:44:40 [INFO] train episode 7: reward = -124.00, steps = 73
06:44:43 [INFO] train episode 8: reward = -504.88, steps = 85
06:44:46 [INFO] train episode 9: reward = -453.93, steps = 79
06:44:48 [INFO] train episode 10: reward = -347.43, steps = 71
06:44:51 [INFO] train episode 11: reward = -244.51, steps = 73
06:44:54 [INFO] train episode 12: reward = -291.91, steps = 73
06:44:56 [INFO] train episode 13: reward = -398.81, steps = 73
06:44:59 [INFO] train episode 14: reward = -281.87, steps = 68
06:45:01 [INFO] train episode 15: reward = -262.03, steps = 53
06:45:03 [INFO] train episode 16: reward = -213.43, steps = 61
06:45:05 [INFO] train episode 17: reward = -308.40, steps = 56
06:45:08 [INFO] train episode 18: reward = -242.01, steps = 76
06:45:13 [INFO] train episode 19: reward = -353.46, steps = 113
06:45:15 [INFO] train episode 20: reward = -209.88, steps = 75
06:45:19 [INFO] train episode 21: reward = -308.92, steps = 98
06:45:22 [INFO] train episode 22: reward = 9.74, steps = 87
06:45:37 [INFO] train episode 23: reward = -520.48, steps = 412
06:45:46 [INFO] train episode 24: reward = -341.28, steps = 229
06:45:51 [INFO] train episode 25: reward = -59.83, steps = 153
06:45:55 [INFO] train episode 26: reward = -122.98, steps = 110
06:46:20 [INFO] train episode 27: reward = -646.65, steps = 668
06:46:55 [INFO] train episode 28: reward = -480.28, steps = 920
06:47:08 [INFO] train episode 29: reward = -272.43, steps = 357
06:47:15 [INFO] train episode 30: reward = -305.03, steps = 173
06:47:28 [INFO] train episode 31: reward = -199.98, steps = 368
06:47:40 [INFO] train episode 32: reward = -124.04, steps = 320
06:47:52 [INFO] train episode 33: reward = -118.74, steps = 317
06:48:04 [INFO] train episode 34: reward = -215.16, steps = 338
06:48:17 [INFO] train episode 35: reward = -232.50, steps = 356
06:48:43 [INFO] train episode 36: reward = -297.74, steps = 693
06:49:02 [INFO] train episode 37: reward = -262.43, steps = 496
06:49:15 [INFO] train episode 38: reward = -269.07, steps = 345
06:49:22 [INFO] train episode 39: reward = -190.48, steps = 191
06:49:31 [INFO] train episode 40: reward = -206.82, steps = 237
06:49:42 [INFO] train episode 41: reward = -196.88, steps = 315
06:49:50 [INFO] train episode 42: reward = -147.69, steps = 200
06:49:58 [INFO] train episode 43: reward = -210.92, steps = 239
06:50:06 [INFO] train episode 44: reward = -169.53, steps = 197
06:50:37 [INFO] train episode 45: reward = -320.28, steps = 840
06:50:48 [INFO] train episode 46: reward = -248.61, steps = 299
06:50:58 [INFO] train episode 47: reward = -239.37, steps = 265
06:51:04 [INFO] train episode 48: reward = -176.54, steps = 167
06:51:36 [INFO] train episode 49: reward = -311.47, steps = 820
06:51:45 [INFO] train episode 50: reward = -147.75, steps = 261
06:52:03 [INFO] train episode 51: reward = -241.43, steps = 471
06:52:10 [INFO] train episode 52: reward = -152.05, steps = 211
06:52:27 [INFO] train episode 53: reward = -226.51, steps = 439
06:52:34 [INFO] train episode 54: reward = -150.29, steps = 188
06:52:44 [INFO] train episode 55: reward = -191.25, steps = 277
06:52:54 [INFO] train episode 56: reward = -156.33, steps = 275
06:53:08 [INFO] train episode 57: reward = -204.69, steps = 385
06:53:19 [INFO] train episode 58: reward = -180.31, steps = 297
06:53:40 [INFO] train episode 59: reward = -190.51, steps = 571
06:53:54 [INFO] train episode 60: reward = -179.89, steps = 364
06:54:16 [INFO] train episode 61: reward = -224.54, steps = 595
06:54:41 [INFO] train episode 62: reward = -255.25, steps = 645
06:55:18 [INFO] train episode 63: reward = -282.85, steps = 984
06:55:36 [INFO] train episode 64: reward = -171.43, steps = 464
06:55:51 [INFO] train episode 65: reward = -202.57, steps = 399
06:56:26 [INFO] train episode 66: reward = -262.52, steps = 910
06:56:36 [INFO] train episode 67: reward = -103.61, steps = 277
06:56:45 [INFO] train episode 68: reward = -117.21, steps = 246
06:57:23 [INFO] train episode 69: reward = -157.12, steps = 1000
06:57:53 [INFO] train episode 70: reward = -251.57, steps = 792
06:58:30 [INFO] train episode 71: reward = -176.52, steps = 1000
06:58:43 [INFO] train episode 72: reward = -154.51, steps = 316
06:59:21 [INFO] train episode 73: reward = -125.18, steps = 1000
06:59:59 [INFO] train episode 74: reward = -91.01, steps = 1000
07:00:39 [INFO] train episode 75: reward = -152.42, steps = 1000
07:01:17 [INFO] train episode 76: reward = -136.96, steps = 1000
07:01:57 [INFO] train episode 77: reward = -155.50, steps = 1000
07:02:35 [INFO] train episode 78: reward = -92.97, steps = 1000
07:03:13 [INFO] train episode 79: reward = -69.89, steps = 1000
07:03:49 [INFO] train episode 80: reward = -103.41, steps = 1000
07:04:26 [INFO] train episode 81: reward = -79.75, steps = 1000
07:05:03 [INFO] train episode 82: reward = -105.82, steps = 1000
07:05:42 [INFO] train episode 83: reward = -136.08, steps = 1000
07:06:23 [INFO] train episode 84: reward = -119.86, steps = 1000
07:07:04 [INFO] train episode 85: reward = -76.90, steps = 1000
07:07:45 [INFO] train episode 86: reward = -97.73, steps = 1000
07:08:25 [INFO] train episode 87: reward = -146.23, steps = 1000
07:09:04 [INFO] train episode 88: reward = -120.31, steps = 1000
07:09:45 [INFO] train episode 89: reward = -113.89, steps = 1000
07:10:26 [INFO] train episode 90: reward = -106.02, steps = 1000
07:11:07 [INFO] train episode 91: reward = -19.21, steps = 1000
07:11:49 [INFO] train episode 92: reward = -116.32, steps = 1000
07:12:29 [INFO] train episode 93: reward = -57.89, steps = 1000
07:13:10 [INFO] train episode 94: reward = -92.17, steps = 1000
07:13:52 [INFO] train episode 95: reward = -122.81, steps = 1000
07:14:33 [INFO] train episode 96: reward = -144.50, steps = 1000
07:15:14 [INFO] train episode 97: reward = -87.08, steps = 1000
07:15:56 [INFO] train episode 98: reward = -125.38, steps = 1000
07:16:37 [INFO] train episode 99: reward = -64.51, steps = 1000
07:17:18 [INFO] train episode 100: reward = -111.63, steps = 1000
07:17:59 [INFO] train episode 101: reward = -52.33, steps = 1000
07:18:43 [INFO] train episode 102: reward = -126.03, steps = 1000
07:19:24 [INFO] train episode 103: reward = -125.11, steps = 1000
07:19:27 [INFO] train episode 104: reward = -205.00, steps = 76
07:20:09 [INFO] train episode 105: reward = -146.56, steps = 1000
07:20:52 [INFO] train episode 106: reward = -139.96, steps = 1000
07:21:32 [INFO] train episode 107: reward = -124.79, steps = 1000
07:22:12 [INFO] train episode 108: reward = -114.53, steps = 1000
07:22:52 [INFO] train episode 109: reward = -95.93, steps = 1000
07:23:35 [INFO] train episode 110: reward = -123.12, steps = 1000
07:24:18 [INFO] train episode 111: reward = -80.41, steps = 1000
07:25:00 [INFO] train episode 112: reward = -76.70, steps = 1000
07:25:44 [INFO] train episode 113: reward = -105.85, steps = 1000
07:26:29 [INFO] train episode 114: reward = -107.08, steps = 1000
07:27:12 [INFO] train episode 115: reward = -129.47, steps = 1000
07:27:57 [INFO] train episode 116: reward = -48.79, steps = 1000
07:28:43 [INFO] train episode 117: reward = -50.67, steps = 1000
07:29:29 [INFO] train episode 118: reward = -99.13, steps = 1000
07:30:15 [INFO] train episode 119: reward = -95.30, steps = 1000
07:31:00 [INFO] train episode 120: reward = -131.54, steps = 1000
07:31:22 [INFO] train episode 121: reward = 234.60, steps = 489
07:32:08 [INFO] train episode 122: reward = -134.29, steps = 1000
07:32:53 [INFO] train episode 123: reward = -63.59, steps = 1000
07:33:26 [INFO] train episode 124: reward = -80.91, steps = 784
07:34:11 [INFO] train episode 125: reward = -115.63, steps = 1000
07:34:54 [INFO] train episode 126: reward = -82.44, steps = 1000
07:35:40 [INFO] train episode 127: reward = -128.58, steps = 1000
07:36:24 [INFO] train episode 128: reward = -67.69, steps = 1000
07:37:09 [INFO] train episode 129: reward = -136.20, steps = 1000
07:37:54 [INFO] train episode 130: reward = -103.51, steps = 1000
07:38:40 [INFO] train episode 131: reward = -146.59, steps = 1000
07:39:25 [INFO] train episode 132: reward = -69.08, steps = 1000
07:40:10 [INFO] train episode 133: reward = -66.15, steps = 1000
07:40:56 [INFO] train episode 134: reward = -125.93, steps = 1000
07:41:43 [INFO] train episode 135: reward = -100.82, steps = 1000
07:42:31 [INFO] train episode 136: reward = -136.78, steps = 1000
07:42:40 [INFO] train episode 137: reward = 1.99, steps = 202
07:42:44 [INFO] train episode 138: reward = -411.09, steps = 92
07:43:29 [INFO] train episode 139: reward = -84.54, steps = 1000
07:44:13 [INFO] train episode 140: reward = -95.32, steps = 1000
07:45:00 [INFO] train episode 141: reward = -64.36, steps = 1000
07:45:46 [INFO] train episode 142: reward = -46.74, steps = 1000
07:46:32 [INFO] train episode 143: reward = -111.39, steps = 1000
07:47:18 [INFO] train episode 144: reward = -133.64, steps = 1000
07:48:05 [INFO] train episode 145: reward = -173.13, steps = 1000
07:48:59 [INFO] train episode 146: reward = -139.49, steps = 1000
07:49:54 [INFO] train episode 147: reward = -110.06, steps = 1000
07:50:42 [INFO] train episode 148: reward = -120.51, steps = 1000
07:51:03 [INFO] train episode 149: reward = -194.49, steps = 449
07:51:50 [INFO] train episode 150: reward = -91.71, steps = 1000
07:52:38 [INFO] train episode 151: reward = -109.07, steps = 1000
07:53:25 [INFO] train episode 152: reward = -120.29, steps = 1000
07:54:12 [INFO] train episode 153: reward = -77.45, steps = 1000
07:55:00 [INFO] train episode 154: reward = -106.42, steps = 1000
07:55:49 [INFO] train episode 155: reward = -135.87, steps = 1000
07:56:37 [INFO] train episode 156: reward = -87.26, steps = 1000
07:57:26 [INFO] train episode 157: reward = -37.32, steps = 1000
07:58:12 [INFO] train episode 158: reward = -108.85, steps = 1000
07:59:00 [INFO] train episode 159: reward = -92.93, steps = 1000
07:59:48 [INFO] train episode 160: reward = -51.78, steps = 1000
08:00:36 [INFO] train episode 161: reward = -94.08, steps = 1000
08:01:29 [INFO] train episode 162: reward = -65.62, steps = 1000
08:02:24 [INFO] train episode 163: reward = -96.11, steps = 1000
08:03:19 [INFO] train episode 164: reward = -72.80, steps = 1000
08:04:07 [INFO] train episode 165: reward = -90.31, steps = 1000
08:04:58 [INFO] train episode 166: reward = -103.97, steps = 1000
08:05:46 [INFO] train episode 167: reward = -110.74, steps = 1000
08:06:34 [INFO] train episode 168: reward = -87.47, steps = 1000
08:07:22 [INFO] train episode 169: reward = -110.01, steps = 1000
08:08:10 [INFO] train episode 170: reward = -72.90, steps = 1000
08:08:57 [INFO] train episode 171: reward = -102.21, steps = 1000
08:09:43 [INFO] train episode 172: reward = -78.22, steps = 1000
08:10:30 [INFO] train episode 173: reward = -134.52, steps = 1000
08:11:17 [INFO] train episode 174: reward = -112.33, steps = 1000
08:12:06 [INFO] train episode 175: reward = -124.90, steps = 1000
08:12:54 [INFO] train episode 176: reward = -104.53, steps = 1000
08:13:41 [INFO] train episode 177: reward = -104.41, steps = 1000
08:13:46 [INFO] train episode 178: reward = -120.70, steps = 117
08:14:34 [INFO] train episode 179: reward = -116.06, steps = 1000
08:15:22 [INFO] train episode 180: reward = -109.05, steps = 1000
08:16:13 [INFO] train episode 181: reward = -138.38, steps = 1000
08:16:57 [INFO] train episode 182: reward = -180.98, steps = 872
08:17:43 [INFO] train episode 183: reward = -121.58, steps = 1000
08:18:31 [INFO] train episode 184: reward = -92.83, steps = 1000
08:19:18 [INFO] train episode 185: reward = -114.65, steps = 1000
09:45:10 [INFO] train episode 186: reward = -91.63, steps = 1000
09:46:00 [INFO] train episode 187: reward = -51.81, steps = 1000
09:46:51 [INFO] train episode 188: reward = -111.85, steps = 1000
09:47:41 [INFO] train episode 189: reward = -86.58, steps = 1000
09:48:33 [INFO] train episode 190: reward = -103.56, steps = 1000
09:49:23 [INFO] train episode 191: reward = -59.68, steps = 1000
09:50:17 [INFO] train episode 192: reward = -103.80, steps = 1000
09:51:12 [INFO] train episode 193: reward = -87.74, steps = 1000
09:52:01 [INFO] train episode 194: reward = -102.89, steps = 1000
09:52:50 [INFO] train episode 195: reward = -71.90, steps = 1000
09:53:40 [INFO] train episode 196: reward = -100.19, steps = 1000
09:54:33 [INFO] train episode 197: reward = -111.32, steps = 1000
09:55:27 [INFO] train episode 198: reward = -130.02, steps = 1000
09:56:19 [INFO] train episode 199: reward = -102.46, steps = 1000
09:57:10 [INFO] train episode 200: reward = -94.77, steps = 1000
09:58:00 [INFO] train episode 201: reward = -95.71, steps = 1000
09:58:50 [INFO] train episode 202: reward = -130.00, steps = 1000
09:59:42 [INFO] train episode 203: reward = -90.24, steps = 1000
10:00:35 [INFO] train episode 204: reward = -56.69, steps = 1000
10:01:24 [INFO] train episode 205: reward = -127.45, steps = 1000
10:02:13 [INFO] train episode 206: reward = -159.15, steps = 1000
10:03:05 [INFO] train episode 207: reward = -89.27, steps = 1000
10:03:57 [INFO] train episode 208: reward = -143.35, steps = 1000
10:04:02 [INFO] train episode 209: reward = -152.49, steps = 107
10:04:15 [INFO] train episode 210: reward = -218.52, steps = 281
10:04:23 [INFO] train episode 211: reward = -132.44, steps = 148
10:05:13 [INFO] train episode 212: reward = -112.33, steps = 1000
10:05:21 [INFO] train episode 213: reward = -190.35, steps = 173
10:05:34 [INFO] train episode 214: reward = -192.30, steps = 250
10:05:43 [INFO] train episode 215: reward = -372.53, steps = 185
10:05:51 [INFO] train episode 216: reward = -448.11, steps = 173
10:05:59 [INFO] train episode 217: reward = -514.03, steps = 155
10:06:04 [INFO] train episode 218: reward = -331.05, steps = 102
10:06:13 [INFO] train episode 219: reward = -355.23, steps = 180
10:06:21 [INFO] train episode 220: reward = -315.29, steps = 164
10:06:27 [INFO] train episode 221: reward = -334.99, steps = 110
10:06:32 [INFO] train episode 222: reward = -338.17, steps = 95
10:06:39 [INFO] train episode 223: reward = -239.06, steps = 134
10:07:33 [INFO] train episode 224: reward = -235.71, steps = 1000
10:07:47 [INFO] train episode 225: reward = -221.32, steps = 285
10:07:55 [INFO] train episode 226: reward = -21.88, steps = 152
10:08:01 [INFO] train episode 227: reward = -191.53, steps = 137
10:08:08 [INFO] train episode 228: reward = -287.15, steps = 146
10:08:12 [INFO] train episode 229: reward = -140.63, steps = 77
10:08:16 [INFO] train episode 230: reward = -123.00, steps = 69
10:08:22 [INFO] train episode 231: reward = -153.94, steps = 135
10:08:35 [INFO] train episode 232: reward = -123.91, steps = 257
10:08:38 [INFO] train episode 233: reward = -85.80, steps = 65
10:08:43 [INFO] train episode 234: reward = -52.43, steps = 95
10:08:51 [INFO] train episode 235: reward = -237.58, steps = 111
10:09:23 [INFO] train episode 236: reward = -77.69, steps = 620
10:09:26 [INFO] train episode 237: reward = -143.91, steps = 59
10:09:52 [INFO] train episode 238: reward = -178.96, steps = 569
10:09:54 [INFO] train episode 239: reward = -57.07, steps = 65
10:10:21 [INFO] train episode 240: reward = 226.20, steps = 582
10:10:26 [INFO] train episode 241: reward = -263.10, steps = 109
10:10:44 [INFO] train episode 242: reward = -118.00, steps = 407
10:10:49 [INFO] train episode 243: reward = -145.33, steps = 111
10:11:19 [INFO] train episode 244: reward = -276.73, steps = 660
10:11:22 [INFO] train episode 245: reward = -78.30, steps = 75
10:11:27 [INFO] train episode 246: reward = -175.75, steps = 113
10:11:46 [INFO] train episode 247: reward = -202.39, steps = 429
10:11:56 [INFO] train episode 248: reward = -87.05, steps = 212
10:11:59 [INFO] train episode 249: reward = -105.88, steps = 76
10:12:03 [INFO] train episode 250: reward = 24.85, steps = 99
10:12:06 [INFO] train episode 251: reward = -31.90, steps = 69
10:12:10 [INFO] train episode 252: reward = -98.10, steps = 88
10:12:13 [INFO] train episode 253: reward = -65.26, steps = 68
10:12:23 [INFO] train episode 254: reward = -47.99, steps = 214
10:12:26 [INFO] train episode 255: reward = -66.61, steps = 78
10:12:30 [INFO] train episode 256: reward = -55.35, steps = 96
10:12:43 [INFO] train episode 257: reward = -108.98, steps = 287
10:12:53 [INFO] train episode 258: reward = -243.54, steps = 228
10:12:58 [INFO] train episode 259: reward = -61.50, steps = 113
10:13:04 [INFO] train episode 260: reward = 6.30, steps = 146
10:13:07 [INFO] train episode 261: reward = -63.49, steps = 69
10:13:12 [INFO] train episode 262: reward = -27.91, steps = 110
10:13:16 [INFO] train episode 263: reward = -28.44, steps = 86
10:13:21 [INFO] train episode 264: reward = -73.21, steps = 125
10:13:33 [INFO] train episode 265: reward = -90.64, steps = 267
10:13:37 [INFO] train episode 266: reward = -71.93, steps = 106
10:13:41 [INFO] train episode 267: reward = -80.66, steps = 83
10:13:46 [INFO] train episode 268: reward = -254.02, steps = 119
10:13:51 [INFO] train episode 269: reward = -175.93, steps = 106
10:13:55 [INFO] train episode 270: reward = -31.64, steps = 101
10:14:00 [INFO] train episode 271: reward = 39.31, steps = 122
10:14:04 [INFO] train episode 272: reward = -96.64, steps = 87
10:14:08 [INFO] train episode 273: reward = -126.36, steps = 98
10:14:11 [INFO] train episode 274: reward = -93.53, steps = 58
10:14:16 [INFO] train episode 275: reward = -29.92, steps = 123
10:14:21 [INFO] train episode 276: reward = -195.91, steps = 105
10:14:25 [INFO] train episode 277: reward = -59.00, steps = 94
10:14:28 [INFO] train episode 278: reward = -69.29, steps = 60
10:14:33 [INFO] train episode 279: reward = -79.33, steps = 115
10:14:39 [INFO] train episode 280: reward = -46.21, steps = 131
10:14:42 [INFO] train episode 281: reward = -41.76, steps = 78
10:14:49 [INFO] train episode 282: reward = -81.62, steps = 167
10:14:56 [INFO] train episode 283: reward = -61.45, steps = 136
10:15:00 [INFO] train episode 284: reward = -83.56, steps = 114
10:15:08 [INFO] train episode 285: reward = -45.95, steps = 182
10:15:13 [INFO] train episode 286: reward = -231.50, steps = 100
10:15:19 [INFO] train episode 287: reward = 2.56, steps = 142
10:15:23 [INFO] train episode 288: reward = -164.72, steps = 91
10:15:28 [INFO] train episode 289: reward = -269.13, steps = 110
10:15:33 [INFO] train episode 290: reward = -187.57, steps = 92
10:15:37 [INFO] train episode 291: reward = -90.53, steps = 90
10:15:44 [INFO] train episode 292: reward = -29.03, steps = 156
10:15:50 [INFO] train episode 293: reward = -79.58, steps = 148
10:15:56 [INFO] train episode 294: reward = -113.31, steps = 134
10:16:01 [INFO] train episode 295: reward = -282.00, steps = 113
10:16:10 [INFO] train episode 296: reward = -286.56, steps = 206
10:16:14 [INFO] train episode 297: reward = -117.19, steps = 98
10:16:20 [INFO] train episode 298: reward = -188.22, steps = 129
10:16:24 [INFO] train episode 299: reward = -178.26, steps = 106
10:16:32 [INFO] train episode 300: reward = -116.22, steps = 167
10:16:36 [INFO] train episode 301: reward = -125.04, steps = 103
10:16:41 [INFO] train episode 302: reward = -100.87, steps = 107
10:16:46 [INFO] train episode 303: reward = -219.16, steps = 108
10:16:53 [INFO] train episode 304: reward = -172.78, steps = 164
10:16:57 [INFO] train episode 305: reward = -224.83, steps = 105
10:17:03 [INFO] train episode 306: reward = 37.84, steps = 120
10:17:12 [INFO] train episode 307: reward = -197.55, steps = 225
10:17:20 [INFO] train episode 308: reward = -244.53, steps = 167
10:17:24 [INFO] train episode 309: reward = 1.84, steps = 98
10:17:28 [INFO] train episode 310: reward = -38.25, steps = 92
10:18:02 [INFO] train episode 311: reward = 99.52, steps = 764
10:18:09 [INFO] train episode 312: reward = -317.85, steps = 157
10:18:15 [INFO] train episode 313: reward = -43.25, steps = 151
10:18:20 [INFO] train episode 314: reward = -31.94, steps = 105
10:18:27 [INFO] train episode 315: reward = -318.80, steps = 161
10:18:35 [INFO] train episode 316: reward = -293.41, steps = 178
10:18:40 [INFO] train episode 317: reward = 19.58, steps = 114
10:18:46 [INFO] train episode 318: reward = -267.28, steps = 152
10:18:53 [INFO] train episode 319: reward = -305.29, steps = 161
10:18:59 [INFO] train episode 320: reward = -165.88, steps = 137
10:19:08 [INFO] train episode 321: reward = -142.77, steps = 203
10:19:24 [INFO] train episode 322: reward = -253.69, steps = 359
10:19:35 [INFO] train episode 323: reward = -215.88, steps = 256
10:19:41 [INFO] train episode 324: reward = -53.77, steps = 139
10:20:26 [INFO] train episode 325: reward = 43.81, steps = 1000
10:20:31 [INFO] train episode 326: reward = 18.88, steps = 126
10:20:47 [INFO] train episode 327: reward = -233.37, steps = 362
10:21:03 [INFO] train episode 328: reward = -96.78, steps = 340
10:21:15 [INFO] train episode 329: reward = -173.47, steps = 268
10:21:34 [INFO] train episode 330: reward = -55.85, steps = 442
10:21:46 [INFO] train episode 331: reward = -61.28, steps = 272
10:22:05 [INFO] train episode 332: reward = -154.27, steps = 408
10:22:22 [INFO] train episode 333: reward = -54.09, steps = 370
10:22:31 [INFO] train episode 334: reward = -12.82, steps = 178
10:22:38 [INFO] train episode 335: reward = -37.68, steps = 162
10:23:24 [INFO] train episode 336: reward = 27.19, steps = 1000
10:23:36 [INFO] train episode 337: reward = -49.52, steps = 266
10:24:19 [INFO] train episode 338: reward = 91.25, steps = 937
10:25:05 [INFO] train episode 339: reward = -29.73, steps = 1000
10:25:50 [INFO] train episode 340: reward = -5.60, steps = 1000
10:26:37 [INFO] train episode 341: reward = -36.03, steps = 1000
10:27:29 [INFO] train episode 342: reward = -87.34, steps = 1000
10:28:18 [INFO] train episode 343: reward = -40.44, steps = 1000
10:28:26 [INFO] train episode 344: reward = -178.64, steps = 180
10:29:11 [INFO] train episode 345: reward = 1.81, steps = 1000
10:29:56 [INFO] train episode 346: reward = 8.41, steps = 1000
10:30:42 [INFO] train episode 347: reward = -37.85, steps = 1000
10:31:30 [INFO] train episode 348: reward = -59.51, steps = 1000
10:32:17 [INFO] train episode 349: reward = -48.58, steps = 1000
10:33:07 [INFO] train episode 350: reward = 19.01, steps = 1000
10:34:03 [INFO] train episode 351: reward = 20.37, steps = 1000
10:35:00 [INFO] train episode 352: reward = -36.50, steps = 1000
10:35:54 [INFO] train episode 353: reward = 6.67, steps = 1000
10:36:49 [INFO] train episode 354: reward = 15.42, steps = 1000
10:37:46 [INFO] train episode 355: reward = -3.12, steps = 1000
10:38:42 [INFO] train episode 356: reward = 12.33, steps = 1000
10:39:41 [INFO] train episode 357: reward = -276.81, steps = 999
10:40:36 [INFO] train episode 358: reward = -29.50, steps = 1000
10:41:34 [INFO] train episode 359: reward = -6.15, steps = 1000
10:42:29 [INFO] train episode 360: reward = 11.12, steps = 1000
10:43:26 [INFO] train episode 361: reward = -46.64, steps = 1000
10:44:21 [INFO] train episode 362: reward = 20.21, steps = 1000
10:45:17 [INFO] train episode 363: reward = -21.80, steps = 1000
10:46:07 [INFO] train episode 364: reward = 4.08, steps = 1000
10:46:58 [INFO] train episode 365: reward = 25.01, steps = 1000
10:47:51 [INFO] train episode 366: reward = -11.14, steps = 1000
10:48:43 [INFO] train episode 367: reward = -5.90, steps = 1000
10:49:34 [INFO] train episode 368: reward = 152.86, steps = 942
10:50:27 [INFO] train episode 369: reward = 98.29, steps = 982
10:51:17 [INFO] train episode 370: reward = 6.46, steps = 1000
10:52:10 [INFO] train episode 371: reward = -4.99, steps = 1000
10:53:02 [INFO] train episode 372: reward = 59.76, steps = 1000
10:53:55 [INFO] train episode 373: reward = 26.03, steps = 1000
10:54:49 [INFO] train episode 374: reward = -128.53, steps = 1000
10:55:42 [INFO] train episode 375: reward = 38.01, steps = 1000
10:56:35 [INFO] train episode 376: reward = 13.31, steps = 1000
10:57:25 [INFO] train episode 377: reward = 41.84, steps = 1000
10:58:05 [INFO] train episode 378: reward = 162.93, steps = 769
10:58:50 [INFO] train episode 379: reward = 115.39, steps = 884
10:59:29 [INFO] train episode 380: reward = 170.95, steps = 732
11:00:13 [INFO] train episode 381: reward = 127.19, steps = 843
11:00:31 [INFO] train episode 382: reward = 293.36, steps = 344
11:01:09 [INFO] train episode 383: reward = 176.53, steps = 734
11:01:47 [INFO] train episode 384: reward = 172.70, steps = 766
11:02:17 [INFO] train episode 385: reward = 202.70, steps = 606
11:02:45 [INFO] train episode 386: reward = 216.61, steps = 529
11:03:13 [INFO] train episode 387: reward = 198.19, steps = 549
11:04:02 [INFO] train episode 388: reward = 145.91, steps = 928
11:04:34 [INFO] train episode 389: reward = 181.54, steps = 575
11:05:01 [INFO] train episode 390: reward = 241.98, steps = 487
11:05:25 [INFO] train episode 391: reward = 248.55, steps = 414
11:05:25 [INFO] ==== test ====
11:05:26 [INFO] test episode 0: reward = 277.28, steps = 418
11:05:27 [INFO] test episode 1: reward = 233.27, steps = 401
11:05:28 [INFO] test episode 2: reward = 228.74, steps = 514
11:05:29 [INFO] test episode 3: reward = 225.76, steps = 503
11:05:30 [INFO] test episode 4: reward = 249.42, steps = 523
11:05:31 [INFO] test episode 5: reward = 221.33, steps = 464
11:05:32 [INFO] test episode 6: reward = 213.58, steps = 434
11:05:33 [INFO] test episode 7: reward = 259.34, steps = 451
11:05:34 [INFO] test episode 8: reward = 215.36, steps = 584
11:05:35 [INFO] test episode 9: reward = 234.53, steps = 401
11:05:36 [INFO] test episode 10: reward = 218.23, steps = 669
11:05:37 [INFO] test episode 11: reward = 263.28, steps = 464
11:05:38 [INFO] test episode 12: reward = 206.59, steps = 458
11:05:39 [INFO] test episode 13: reward = 263.12, steps = 371
11:05:40 [INFO] test episode 14: reward = 234.53, steps = 449
11:05:40 [INFO] test episode 15: reward = 254.30, steps = 375
11:05:41 [INFO] test episode 16: reward = 224.86, steps = 513
11:05:42 [INFO] test episode 17: reward = 214.34, steps = 489
11:05:43 [INFO] test episode 18: reward = 234.87, steps = 418
11:05:44 [INFO] test episode 19: reward = 208.02, steps = 429
11:05:45 [INFO] test episode 20: reward = 273.05, steps = 384
11:05:46 [INFO] test episode 21: reward = 209.81, steps = 502
11:05:47 [INFO] test episode 22: reward = 221.46, steps = 690
11:05:48 [INFO] test episode 23: reward = 211.70, steps = 624
11:05:49 [INFO] test episode 24: reward = 221.82, steps = 379
11:05:50 [INFO] test episode 25: reward = 211.89, steps = 598
11:05:52 [INFO] test episode 26: reward = 214.95, steps = 489
11:05:52 [INFO] test episode 27: reward = 255.48, steps = 374
11:05:54 [INFO] test episode 28: reward = 222.00, steps = 506
11:05:54 [INFO] test episode 29: reward = 234.99, steps = 420
11:05:55 [INFO] test episode 30: reward = 270.02, steps = 358
11:05:56 [INFO] test episode 31: reward = 274.00, steps = 369
11:05:56 [INFO] test episode 32: reward = 197.17, steps = 421
11:05:57 [INFO] test episode 33: reward = 265.64, steps = 493
11:05:58 [INFO] test episode 34: reward = 242.44, steps = 423
11:06:00 [INFO] test episode 35: reward = 218.41, steps = 621
11:06:01 [INFO] test episode 36: reward = 193.90, steps = 437
11:06:01 [INFO] test episode 37: reward = 242.15, steps = 433
11:06:02 [INFO] test episode 38: reward = 253.12, steps = 418
11:06:04 [INFO] test episode 39: reward = 219.34, steps = 561
11:06:05 [INFO] test episode 40: reward = 248.94, steps = 462
11:06:05 [INFO] test episode 41: reward = 254.59, steps = 400
11:06:06 [INFO] test episode 42: reward = 204.38, steps = 464
11:06:07 [INFO] test episode 43: reward = 227.20, steps = 411
11:06:08 [INFO] test episode 44: reward = 246.14, steps = 426
11:06:09 [INFO] test episode 45: reward = 240.90, steps = 406
11:06:10 [INFO] test episode 46: reward = 221.85, steps = 638
11:06:11 [INFO] test episode 47: reward = 223.06, steps = 426
11:06:12 [INFO] test episode 48: reward = 266.92, steps = 477
11:06:13 [INFO] test episode 49: reward = 241.36, steps = 537
11:06:14 [INFO] test episode 50: reward = 213.25, steps = 392
11:06:15 [INFO] test episode 51: reward = 238.72, steps = 432
11:06:15 [INFO] test episode 52: reward = 270.38, steps = 400
11:06:16 [INFO] test episode 53: reward = 231.75, steps = 385
11:06:17 [INFO] test episode 54: reward = 247.63, steps = 424
11:06:18 [INFO] test episode 55: reward = 231.26, steps = 412
11:06:19 [INFO] test episode 56: reward = 202.18, steps = 403
11:06:19 [INFO] test episode 57: reward = 212.22, steps = 447
11:06:20 [INFO] test episode 58: reward = 261.40, steps = 449
11:06:21 [INFO] test episode 59: reward = 224.69, steps = 450
11:06:22 [INFO] test episode 60: reward = 264.63, steps = 427
11:06:23 [INFO] test episode 61: reward = 231.54, steps = 480
11:06:24 [INFO] test episode 62: reward = 237.32, steps = 425
11:06:25 [INFO] test episode 63: reward = 229.52, steps = 435
11:06:25 [INFO] test episode 64: reward = 257.12, steps = 421
11:06:26 [INFO] test episode 65: reward = 228.47, steps = 444
11:06:27 [INFO] test episode 66: reward = 194.83, steps = 432
11:06:28 [INFO] test episode 67: reward = 210.60, steps = 504
11:06:30 [INFO] test episode 68: reward = 216.76, steps = 672
11:06:31 [INFO] test episode 69: reward = 258.72, steps = 449
11:06:32 [INFO] test episode 70: reward = 232.74, steps = 385
11:06:33 [INFO] test episode 71: reward = 258.31, steps = 448
11:06:34 [INFO] test episode 72: reward = 250.08, steps = 492
11:06:35 [INFO] test episode 73: reward = 207.09, steps = 453
11:06:36 [INFO] test episode 74: reward = 239.58, steps = 494
11:06:36 [INFO] test episode 75: reward = 283.37, steps = 398
11:06:37 [INFO] test episode 76: reward = 273.23, steps = 406
11:06:38 [INFO] test episode 77: reward = 199.14, steps = 437
11:06:39 [INFO] test episode 78: reward = 206.96, steps = 453
11:06:40 [INFO] test episode 79: reward = 232.77, steps = 527
11:06:41 [INFO] test episode 80: reward = 262.82, steps = 414
11:06:42 [INFO] test episode 81: reward = 238.25, steps = 417
11:06:42 [INFO] test episode 82: reward = 213.53, steps = 486
11:06:43 [INFO] test episode 83: reward = 276.50, steps = 429
11:06:44 [INFO] test episode 84: reward = 221.97, steps = 602
11:06:46 [INFO] test episode 85: reward = 210.29, steps = 722
11:06:46 [INFO] test episode 86: reward = 248.47, steps = 399
11:06:48 [INFO] test episode 87: reward = 218.88, steps = 714
11:06:49 [INFO] test episode 88: reward = 197.18, steps = 432
11:06:50 [INFO] test episode 89: reward = 229.81, steps = 405
11:06:51 [INFO] test episode 90: reward = 228.87, steps = 470
11:06:52 [INFO] test episode 91: reward = 243.35, steps = 437
11:06:53 [INFO] test episode 92: reward = 227.23, steps = 441
11:06:54 [INFO] test episode 93: reward = 252.32, steps = 409
11:06:55 [INFO] test episode 94: reward = 251.28, steps = 409
11:06:56 [INFO] test episode 95: reward = 210.83, steps = 484
11:06:57 [INFO] test episode 96: reward = 222.87, steps = 490
11:06:58 [INFO] test episode 97: reward = 209.81, steps = 603
11:06:59 [INFO] test episode 98: reward = 277.47, steps = 399
11:07:00 [INFO] test episode 99: reward = 217.30, steps = 448
11:07:00 [INFO] average episode reward = 234.15 ± 22.26

env.close()