Refactoring lab
This commit is contained in:
Chris Proctor
@@ -1,3 +0,0 @@
|
||||
from babysnake import create_game
|
||||
|
||||
create_game().play()
|
||||
@@ -1,4 +0,0 @@
|
||||
[tool.retro-gamer]
|
||||
actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"]
|
||||
reward = "reward"
|
||||
observation_function = "babysnake:get_state"
|
||||
@@ -1,111 +0,0 @@
|
||||
"""Forager: an 8×8 grid game where an agent collects food.
|
||||
|
||||
The agent moves in four directions collecting food that respawns on collection.
|
||||
The agent starts with 100 energy. Each step costs 1 energy. Collecting food
|
||||
restores 40 energy. The episode ends when energy reaches 0.
|
||||
|
||||
Observation features for retro-gamer:
|
||||
food_dx: (food_x - agent_x) / board_width (positive = food is to the right)
|
||||
food_dy: (food_y - agent_y) / board_height (positive = food is below)
|
||||
"""
|
||||
|
||||
from random import randint
|
||||
from retro.game import Game
|
||||
|
||||
BOARD_SIZE = 8
|
||||
START_ENERGY = 100
|
||||
FOOD_ENERGY = 40
|
||||
|
||||
|
||||
class Forager:
|
||||
"""The player agent."""
|
||||
name = "Forager"
|
||||
character = '@'
|
||||
color = "green_on_black"
|
||||
position = (0, 0)
|
||||
|
||||
UP = (0, -1)
|
||||
DOWN = (0, 1)
|
||||
LEFT = (-1, 0)
|
||||
RIGHT = (1, 0)
|
||||
|
||||
def __init__(self):
|
||||
self._direction = self.RIGHT
|
||||
|
||||
def handle_keystroke(self, keystroke, game):
|
||||
if keystroke.name == "KEY_RIGHT":
|
||||
self._direction = self.RIGHT
|
||||
elif keystroke.name == "KEY_UP":
|
||||
self._direction = self.UP
|
||||
elif keystroke.name == "KEY_LEFT":
|
||||
self._direction = self.LEFT
|
||||
elif keystroke.name == "KEY_DOWN":
|
||||
self._direction = self.DOWN
|
||||
|
||||
def play_turn(self, game):
|
||||
x, y = self.position
|
||||
dx, dy = self._direction
|
||||
new_pos = (x + dx, y + dy)
|
||||
if game.on_board(new_pos):
|
||||
self.position = new_pos
|
||||
|
||||
game.state['energy'] -= 1
|
||||
game.state['reward'] -= 0.01
|
||||
|
||||
food = game.get_agent_by_name("Food")
|
||||
if self.position == food.position:
|
||||
food.relocate(game)
|
||||
game.state['energy'] += FOOD_ENERGY
|
||||
game.state['score'] += 1
|
||||
game.state['reward'] += 1.0
|
||||
|
||||
bw, bh = game.board_size
|
||||
ax, ay = self.position
|
||||
fx, fy = game.get_agent_by_name("Food").position
|
||||
game.state['food_dx'] = (fx - ax) / bw
|
||||
game.state['food_dy'] = (fy - ay) / bh
|
||||
|
||||
if game.state['energy'] <= 0:
|
||||
game.end()
|
||||
|
||||
|
||||
class Food:
|
||||
"""The food item. Respawns when collected."""
|
||||
name = "Food"
|
||||
character = '*'
|
||||
color = "yellow_on_black"
|
||||
position = (0, 0)
|
||||
|
||||
def relocate(self, game):
|
||||
bw, bh = game.board_size
|
||||
forager = game.get_agent_by_name("Forager")
|
||||
while True:
|
||||
pos = (randint(0, bw - 1), randint(0, bh - 1))
|
||||
if pos != forager.position:
|
||||
self.position = pos
|
||||
return
|
||||
|
||||
|
||||
def create_game():
|
||||
"""Return a fresh Forager game."""
|
||||
forager = Forager()
|
||||
food = Food()
|
||||
bw = bh = BOARD_SIZE
|
||||
game = Game(
|
||||
[forager, food],
|
||||
{'score': 0, 'reward': 0.0, 'energy': START_ENERGY, 'food_dx': 0.0, 'food_dy': 0.0},
|
||||
board_size=(bw, bh),
|
||||
framerate=12,
|
||||
)
|
||||
forager.position = (randint(0, bw - 1), randint(0, bh - 1))
|
||||
food.relocate(game)
|
||||
bw, bh = game.board_size
|
||||
ax, ay = forager.position
|
||||
fx, fy = food.position
|
||||
game.state['food_dx'] = (fx - ax) / bw
|
||||
game.state['food_dy'] = (fy - ay) / bh
|
||||
return game
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
create_game().play()
|
||||
@@ -1,3 +0,0 @@
|
||||
from forager import create_game
|
||||
|
||||
create_game().play()
|
||||
@@ -1,4 +0,0 @@
|
||||
[tool.retro-gamer]
|
||||
actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"]
|
||||
reward = "reward"
|
||||
character_set = ["@", "*"]
|
||||
0
games/__init__.py
Normal file
0
games/__init__.py
Normal file
3
games/babysnake/__main__.py
Normal file
3
games/babysnake/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from games.babysnake import create_game
|
||||
|
||||
create_game().play()
|
||||
11
games/babysnake/pyproject.toml
Normal file
11
games/babysnake/pyproject.toml
Normal file
@@ -0,0 +1,11 @@
|
||||
[project]
|
||||
name = "babysnake"
|
||||
version = "0.1.0"
|
||||
description = "BabySnake: a tiny grid game for teaching Q-learning"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = ["retro-games>=2.5.0"]
|
||||
|
||||
[tool.retro-gamer]
|
||||
actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"]
|
||||
reward = "reward"
|
||||
observation_function = "games.babysnake:get_state"
|
||||
@@ -10,13 +10,13 @@ babysnake/pyproject.toml's observation_function) as the environment.
|
||||
"""
|
||||
|
||||
from q_learning import QLearning
|
||||
import babysnake
|
||||
from games import babysnake
|
||||
from retro.input import ProgrammaticInput
|
||||
from retro_gamer import GameEnvironment, GameMetadata
|
||||
|
||||
def train():
|
||||
trainer = QLearning()
|
||||
env = GameEnvironment(babysnake.create_game, GameMetadata.from_pyproject("babysnake"))
|
||||
env = GameEnvironment(babysnake.create_game, GameMetadata.from_pyproject("games.babysnake"))
|
||||
return trainer.train(env, babysnake.ACTIONS)
|
||||
|
||||
def watch(Q):
|
||||
161
games/frogger/__init__.py
Normal file
161
games/frogger/__init__.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""Frogger: guide a frog across busy traffic lanes to reach the far side.
|
||||
|
||||
The frog starts at the bottom row. Cars move across the lanes between the
|
||||
start and goal. Each lane has cars moving at different speeds in alternating
|
||||
directions. The frog earns +10 for each row advanced, +50 for reaching the
|
||||
top row, and -10 for being hit by a car or falling off the edge. Episodes end
|
||||
when the frog reaches the top, gets hit, or energy runs out.
|
||||
"""
|
||||
|
||||
from random import randint
|
||||
from retro.game import Game
|
||||
|
||||
BOARD_WIDTH = 20
|
||||
BOARD_HEIGHT = 12
|
||||
NUM_LANES = BOARD_HEIGHT - 2
|
||||
START_ENERGY = 200
|
||||
|
||||
|
||||
class Frog:
|
||||
name = "Frog"
|
||||
character = 'O'
|
||||
color = "green_on_black"
|
||||
position = (0, 0)
|
||||
|
||||
UP = (0, -1)
|
||||
DOWN = (0, 1)
|
||||
LEFT = (-1, 0)
|
||||
RIGHT = (1, 0)
|
||||
|
||||
def __init__(self):
|
||||
self._direction = self.UP
|
||||
|
||||
def handle_keystroke(self, keystroke, game):
|
||||
if keystroke.name == "KEY_UP":
|
||||
self._direction = self.UP
|
||||
elif keystroke.name == "KEY_DOWN":
|
||||
self._direction = self.DOWN
|
||||
elif keystroke.name == "KEY_LEFT":
|
||||
self._direction = self.LEFT
|
||||
elif keystroke.name == "KEY_RIGHT":
|
||||
self._direction = self.RIGHT
|
||||
|
||||
def play_turn(self, game):
|
||||
bw, bh = game.board_size
|
||||
x, y = self.position
|
||||
dx, dy = self._direction
|
||||
nx, ny = x + dx, y + dy
|
||||
|
||||
if not (0 <= nx < bw):
|
||||
game.state['reward'] -= 10
|
||||
game.state['energy'] -= 50
|
||||
self._reset(game)
|
||||
return
|
||||
|
||||
if not (0 <= ny < bh):
|
||||
if ny < 0:
|
||||
game.state['score'] += 50
|
||||
game.state['reward'] += 50
|
||||
else:
|
||||
game.state['reward'] -= 5
|
||||
self._reset(game)
|
||||
return
|
||||
|
||||
prev_y = y
|
||||
self.position = (nx, ny)
|
||||
game.state['energy'] -= 1
|
||||
game.state['reward'] -= 0.01
|
||||
|
||||
if ny < prev_y:
|
||||
advancement = prev_y - ny
|
||||
game.state['score'] += advancement * 10
|
||||
game.state['reward'] += advancement * 5
|
||||
|
||||
for agent in game.agents:
|
||||
if hasattr(agent, '_is_car') and agent.position == self.position:
|
||||
game.state['reward'] -= 10
|
||||
game.state['energy'] -= 50
|
||||
self._reset(game)
|
||||
return
|
||||
|
||||
bw, bh = game.board_size
|
||||
fx, fy = self.position
|
||||
game.state['frog_x'] = fx / bw
|
||||
game.state['frog_y'] = fy / bh
|
||||
|
||||
if game.state['energy'] <= 0:
|
||||
game.end()
|
||||
|
||||
def _reset(self, game):
|
||||
bw, bh = game.board_size
|
||||
self.position = (bw // 2, bh - 1)
|
||||
self._direction = self.UP
|
||||
if game.state['energy'] <= 0:
|
||||
game.end()
|
||||
|
||||
|
||||
class Car:
|
||||
_is_car = True
|
||||
character = 'X'
|
||||
color = "red_on_black"
|
||||
|
||||
def __init__(self, lane, speed, direction, start_x, board_width):
|
||||
self.name = f"Car {lane}_{start_x}"
|
||||
self._lane = lane
|
||||
self._speed = speed
|
||||
self._direction = direction
|
||||
self._board_width = board_width
|
||||
self._step = 0
|
||||
self.position = (start_x, lane)
|
||||
|
||||
def play_turn(self, game):
|
||||
self._step += 1
|
||||
if self._step < self._speed:
|
||||
return
|
||||
self._step = 0
|
||||
x, y = self.position
|
||||
x = (x + self._direction) % self._board_width
|
||||
self.position = (x, y)
|
||||
|
||||
frog = game.get_agent_by_name("Frog")
|
||||
if frog.position == self.position:
|
||||
frog._reset(game)
|
||||
game.state['reward'] -= 10
|
||||
game.state['energy'] -= 50
|
||||
|
||||
|
||||
def create_game():
|
||||
bw, bh = BOARD_WIDTH, BOARD_HEIGHT
|
||||
frog = Frog()
|
||||
frog.position = (bw // 2, bh - 1)
|
||||
|
||||
agents = [frog]
|
||||
car_id = 0
|
||||
for lane_idx, row in enumerate(range(1, bh - 1)):
|
||||
direction = 1 if lane_idx % 2 == 0 else -1
|
||||
speed = 2 + (lane_idx % 3)
|
||||
num_cars = 2 + (lane_idx % 3)
|
||||
spacing = bw // num_cars
|
||||
for i in range(num_cars):
|
||||
start_x = (i * spacing + lane_idx * 3) % bw
|
||||
agents.append(Car(row, speed, direction, start_x, bw))
|
||||
car_id += 1
|
||||
|
||||
game = Game(
|
||||
agents,
|
||||
{
|
||||
'score': 0,
|
||||
'reward': 0.0,
|
||||
'energy': START_ENERGY,
|
||||
'frog_x': (bw // 2) / bw,
|
||||
'frog_y': (bh - 1) / bh,
|
||||
},
|
||||
board_size=(bw, bh),
|
||||
framerate=8,
|
||||
show_state=['score', 'energy'],
|
||||
)
|
||||
return game
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
create_game().play()
|
||||
3
games/frogger/__main__.py
Normal file
3
games/frogger/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from games.frogger import create_game
|
||||
|
||||
create_game().play()
|
||||
11
games/frogger/pyproject.toml
Normal file
11
games/frogger/pyproject.toml
Normal file
@@ -0,0 +1,11 @@
|
||||
[project]
|
||||
name = "frogger"
|
||||
version = "0.1.0"
|
||||
description = "Frogger: guide a frog across traffic lanes to train an RL agent"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = ["retro-games>=2.5.0"]
|
||||
|
||||
[tool.retro-gamer]
|
||||
actions = ["KEY_UP", "KEY_DOWN", "KEY_LEFT", "KEY_RIGHT"]
|
||||
reward = "reward"
|
||||
character_set = ["O", "X"]
|
||||
36
games/snake/__init__.py
Normal file
36
games/snake/__init__.py
Normal file
@@ -0,0 +1,36 @@
|
||||
from random import randint, choice
|
||||
from .snake import SnakeHead
|
||||
from .apple import Apple
|
||||
from retro.game import Game
|
||||
|
||||
def create_game():
|
||||
"""Return a fresh, initialized Snake game."""
|
||||
head = SnakeHead()
|
||||
apple = Apple()
|
||||
game = Game(
|
||||
[head, apple],
|
||||
{'score': 0, 'reward': 0, 'energy': 150, 'apple_dx': 0.0, 'apple_dy': 0.0},
|
||||
board_size=(32, 16),
|
||||
framerate=12,
|
||||
)
|
||||
bw, bh = game.board_size
|
||||
head.position = (randint(1, bw - 2), randint(1, bh - 2))
|
||||
direction, character = choice([
|
||||
(SnakeHead.RIGHT, '>'),
|
||||
(SnakeHead.UP, '^'),
|
||||
(SnakeHead.LEFT, '<'),
|
||||
(SnakeHead.DOWN, 'v'),
|
||||
])
|
||||
head.direction = direction
|
||||
head.character = character
|
||||
apple.relocate(game)
|
||||
hx, hy = head.position
|
||||
ax, ay = apple.position
|
||||
game.state['apple_dx'] = (ax - hx) / bw
|
||||
game.state['apple_dy'] = (ay - hy) / bh
|
||||
return game
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
create_game().play()
|
||||
|
||||
3
games/snake/__main__.py
Normal file
3
games/snake/__main__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from games.snake import create_game
|
||||
|
||||
create_game().play()
|
||||
49
games/snake/apple.py
Normal file
49
games/snake/apple.py
Normal file
@@ -0,0 +1,49 @@
|
||||
from random import randint
|
||||
|
||||
class Apple:
|
||||
"""An agent representing the Apple.
|
||||
Note how Apple doesn't have ``play_turn`` or
|
||||
``handle_keystroke`` methods: the Apple doesn't need to do
|
||||
anything in this game. It just sits there waiting to get
|
||||
eaten.
|
||||
|
||||
Attributes:
|
||||
name: "Apple"
|
||||
character: '@'
|
||||
color: "red_on_black" (`Here's documentation on how colors
|
||||
work <https://blessed.readthedocs.io/en/latest/colors.html>`_
|
||||
position: (0, 0). The Apple will choose a random position
|
||||
as soon as the game starts, but it needs an initial
|
||||
position to be assigned.
|
||||
|
||||
"""
|
||||
name = "Apple"
|
||||
character = '@'
|
||||
color = "red_on_black"
|
||||
position = (0, 0)
|
||||
|
||||
def relocate(self, game):
|
||||
"""Sets position to a random empty position. This method is
|
||||
called whenever the snake's head touches the apple.
|
||||
|
||||
Arguments:
|
||||
game (Game): The current game.
|
||||
"""
|
||||
self.position = self.random_empty_position(game)
|
||||
|
||||
def random_empty_position(self, game):
|
||||
"""Returns a randomly-selected empty position. Uses a very
|
||||
simple algorithm: Get the game's board size, choose a
|
||||
random x-value between 0 and the board width, and choose
|
||||
a random y-value between 0 and the board height. Now use
|
||||
the game to check whether any Agents are occupying this
|
||||
position. If so, keep randomly choosing a new position
|
||||
until the position is empty.
|
||||
"""
|
||||
bw, bh = game.board_size
|
||||
occupied_positions = game.get_agents_by_position()
|
||||
while True:
|
||||
position = (randint(0, bw-1), randint(0, bh-1))
|
||||
if position not in occupied_positions:
|
||||
return position
|
||||
|
||||
18
games/snake/observation.py
Normal file
18
games/snake/observation.py
Normal file
@@ -0,0 +1,18 @@
|
||||
import numpy as np
|
||||
from retro.views.headless import HeadlessView
|
||||
from retro_gamer.observation import egocentric_board, encode_board, encode_state
|
||||
|
||||
CHARACTER_SET = ["@", "*", ">", "<", "^", "v"]
|
||||
RADIUS = 8
|
||||
|
||||
|
||||
def egocentric_observation(game):
|
||||
"""17×17 window centered on the snake's head, plus apple_dx and apple_dy."""
|
||||
view = HeadlessView()
|
||||
view.on_game_start(game)
|
||||
view.render(game)
|
||||
head = game.get_agent_by_name("Snake head")
|
||||
cropped = egocentric_board(view.board_characters, head.position, RADIUS)
|
||||
board_vec = encode_board(cropped, CHARACTER_SET).flatten()
|
||||
extras = encode_state(game.state, ["apple_dx", "apple_dy"])
|
||||
return np.concatenate([board_vec, extras])
|
||||
10
games/snake/pyproject.toml
Normal file
10
games/snake/pyproject.toml
Normal file
@@ -0,0 +1,10 @@
|
||||
[project]
|
||||
name = "snake"
|
||||
version = "0.1.0"
|
||||
description = "Snake: a classic game for training RL agents"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = ["retro-games>=2.5.0"]
|
||||
|
||||
[tool.retro-gamer]
|
||||
actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"]
|
||||
reward = "reward"
|
||||
160
games/snake/snake.py
Normal file
160
games/snake/snake.py
Normal file
@@ -0,0 +1,160 @@
|
||||
|
||||
class SnakeHead:
|
||||
"""An Agent representing the snake's head. When the game starts, you control
|
||||
the snake head using the arrow keys. The SnakeHead always has a direction, and
|
||||
will keep moving in that direction every turn. When you press an arrow key,
|
||||
you change the SnakeHead's direction.
|
||||
|
||||
Attributes:
|
||||
name: "Snake head"
|
||||
position: (0,0)
|
||||
character: ``'v'`` Depending on the snake head's direction, its character
|
||||
changes to ``'<'``, ``'^'``, ``'>'``, or ``'v'``.
|
||||
next_segment: Initially ``None``, this is a reference to a SnakeBodySegment.
|
||||
growing: When set to True, the snake will grow a new segment on its next move.
|
||||
"""
|
||||
RIGHT = (1, 0)
|
||||
UP = (0, -1)
|
||||
LEFT = (-1, 0)
|
||||
DOWN = (0, 1)
|
||||
name = "Snake head"
|
||||
position = (0, 0)
|
||||
direction = DOWN
|
||||
character = 'v'
|
||||
next_segment = None
|
||||
growing = False
|
||||
|
||||
def play_turn(self, game):
|
||||
"""On each turn, the snake head uses its position and direction to figure out
|
||||
its next position. If the snake head is able to move there (it's on the board and
|
||||
not occuppied by part of the snake's body), it moves.
|
||||
|
||||
Then, if the snake head is on the Apple, the Apple moves to a new random position
|
||||
and ``growing`` is set to True.
|
||||
|
||||
Now we need to deal with two situations. First, if ``next_segment`` is not None, there is
|
||||
a SnakeBodySegment attached to the head. We need the body to follow the head,
|
||||
so we call ``self.next_segment.move``, passing the head's old position
|
||||
(this will be the body's new position), a reference to the game, and a value for
|
||||
``growing``. If the snake needs to grow, we need to pass this information along
|
||||
the body until it reaches the tail--this is where the next segment will be attached.
|
||||
|
||||
If there is no ``next_segment`` but ``self.growing`` is True, it's time to add
|
||||
a body! We set ``self.next_segment`` to a new SnakeBodySegment, set its
|
||||
position to the head's old position, and add it to the game. We also add 1 to the
|
||||
game's score.
|
||||
"""
|
||||
x, y = self.position
|
||||
dx, dy = self.direction
|
||||
next_pos = (x + dx, y + dy)
|
||||
if self.can_move(next_pos, game):
|
||||
apple = game.get_agent_by_name("Apple")
|
||||
ax, ay = apple.position
|
||||
old_dist = abs(x - ax) + abs(y - ay)
|
||||
new_dist = abs(next_pos[0] - ax) + abs(next_pos[1] - ay)
|
||||
game.state['reward'] += old_dist - new_dist # +1 toward apple, -1 away
|
||||
game.state['energy'] -= 1
|
||||
self.position = next_pos
|
||||
if self.is_on_apple(self.position, game):
|
||||
apple.relocate(game)
|
||||
self.growing = True
|
||||
game.state['score'] += 50
|
||||
game.state['reward'] += 50
|
||||
game.state['energy'] = 150
|
||||
if self.next_segment:
|
||||
self.next_segment.move((x, y), game, growing=self.growing)
|
||||
elif self.growing:
|
||||
self.next_segment = SnakeBodySegment(1, (x, y))
|
||||
game.add_agent(self.next_segment)
|
||||
self.growing = False
|
||||
bw, bh = game.board_size
|
||||
hx, hy = self.position
|
||||
ax, ay = apple.position
|
||||
game.state['apple_dx'] = (ax - hx) / bw
|
||||
game.state['apple_dy'] = (ay - hy) / bh
|
||||
if game.state['energy'] <= 0:
|
||||
game.state['reward'] -= 10
|
||||
game.end()
|
||||
else:
|
||||
game.state['reward'] -= 10
|
||||
game.end()
|
||||
|
||||
def handle_keystroke(self, keystroke, game):
|
||||
"""Checks whether one of the arrow keys has been pressed.
|
||||
If so, sets the SnakeHead's direction and character.
|
||||
"""
|
||||
x, y = self.position
|
||||
if keystroke.name == "KEY_RIGHT":
|
||||
self.direction = self.RIGHT
|
||||
self.character = '>'
|
||||
elif keystroke.name == "KEY_UP":
|
||||
self.direction = self.UP
|
||||
self.character = '^'
|
||||
elif keystroke.name == "KEY_LEFT":
|
||||
self.direction = self.LEFT
|
||||
self.character = '<'
|
||||
elif keystroke.name == "KEY_DOWN":
|
||||
self.direction = self.DOWN
|
||||
self.character = 'v'
|
||||
|
||||
def can_move(self, position, game):
|
||||
on_board = game.on_board(position)
|
||||
empty = game.is_empty(position)
|
||||
on_apple = self.is_on_apple(position, game)
|
||||
return on_board and (empty or on_apple)
|
||||
|
||||
def is_on_apple(self, position, game):
|
||||
apple = game.get_agent_by_name("Apple")
|
||||
return apple.position == position
|
||||
|
||||
class SnakeBodySegment:
|
||||
"""Finally, we need an Agent for the snake's body segments.
|
||||
SnakeBodySegment doesn't have ``play_turn`` or ``handle_keystroke`` methods because
|
||||
it never does anything on its own. It only moves when the SnakeHead, or the previous
|
||||
segment, tells it to move.
|
||||
|
||||
Arguments:
|
||||
segment_id (int): Keeps track of how far back this segment is from the head.
|
||||
This is used to give the segment a unique name, and also to keep track
|
||||
of how many points the player earns for eating the next apple.
|
||||
position (int, int): The initial position.
|
||||
|
||||
Attributes:
|
||||
character: '*'
|
||||
next_segment: Initially ``None``, this is a reference to a SnakeBodySegment
|
||||
when this segment is not the last one in the snake's body.
|
||||
|
||||
"""
|
||||
character = '*'
|
||||
next_segment = None
|
||||
|
||||
def __init__(self, segment_id, position):
|
||||
self.segment_id = segment_id
|
||||
self.name = f"Snake body segment {segment_id}"
|
||||
self.position = position
|
||||
|
||||
def move(self, new_position, game, growing=False):
|
||||
"""When SnakeHead moves, it sets off a chain reaction, moving all its
|
||||
body segments. Whenever the head or a body segment has another segment
|
||||
(``next_segment``), it calls that segment's ``move`` method.
|
||||
|
||||
This method updates the SnakeBodySegment's position. Then, if
|
||||
``self.next_segment`` is not None, calls that segment's ``move`` method.
|
||||
If there is no next segment and ``growing`` is True, then we set
|
||||
``self.next_segment`` to a new SnakeBodySegment in this segment's old
|
||||
position, and update the game's score.
|
||||
|
||||
Arguments:
|
||||
new_position (int, int): The new position.
|
||||
game (Game): A reference to the current game.
|
||||
growing (bool): (Default False) When True, the snake needs to
|
||||
add a new segment.
|
||||
"""
|
||||
old_position = self.position
|
||||
self.position = new_position
|
||||
if self.next_segment:
|
||||
self.next_segment.move(old_position, game, growing=growing)
|
||||
elif growing:
|
||||
self.next_segment = SnakeBodySegment(self.segment_id + 1, old_position)
|
||||
game.add_agent(self.next_segment)
|
||||
|
||||
92
games/snake_v1/__init__.py
Normal file
92
games/snake_v1/__init__.py
Normal file
@@ -0,0 +1,92 @@
|
||||
from random import randint, choice
|
||||
from games.snake.apple import Apple
|
||||
from games.snake.snake import SnakeBodySegment
|
||||
from retro.game import Game
|
||||
|
||||
|
||||
class SnakeHead:
|
||||
RIGHT = (1, 0)
|
||||
UP = (0, -1)
|
||||
LEFT = (-1, 0)
|
||||
DOWN = (0, 1)
|
||||
name = "Snake head"
|
||||
position = (0, 0)
|
||||
direction = DOWN
|
||||
character = 'v'
|
||||
next_segment = None
|
||||
growing = False
|
||||
|
||||
def play_turn(self, game):
|
||||
x, y = self.position
|
||||
dx, dy = self.direction
|
||||
next_pos = (x + dx, y + dy)
|
||||
if self.can_move(next_pos, game):
|
||||
apple = game.get_agent_by_name("Apple")
|
||||
ax, ay = apple.position
|
||||
old_dist = abs(x - ax) + abs(y - ay)
|
||||
new_dist = abs(next_pos[0] - ax) + abs(next_pos[1] - ay)
|
||||
game.state['reward'] += old_dist - new_dist
|
||||
game.state['energy'] -= 1
|
||||
self.position = next_pos
|
||||
if self.is_on_apple(self.position, game):
|
||||
apple.relocate(game)
|
||||
self.growing = True
|
||||
game.state['score'] += 50
|
||||
game.state['reward'] += 50
|
||||
game.state['energy'] = 150
|
||||
if self.next_segment:
|
||||
self.next_segment.move((x, y), game, growing=self.growing)
|
||||
elif self.growing:
|
||||
self.next_segment = SnakeBodySegment(1, (x, y))
|
||||
game.add_agent(self.next_segment)
|
||||
self.growing = False
|
||||
if game.state['energy'] <= 0:
|
||||
game.state['reward'] -= 10
|
||||
game.end()
|
||||
else:
|
||||
game.state['reward'] -= 10
|
||||
game.end()
|
||||
|
||||
def handle_keystroke(self, keystroke, game):
|
||||
if keystroke.name == "KEY_RIGHT":
|
||||
self.direction = self.RIGHT
|
||||
self.character = '>'
|
||||
elif keystroke.name == "KEY_UP":
|
||||
self.direction = self.UP
|
||||
self.character = '^'
|
||||
elif keystroke.name == "KEY_LEFT":
|
||||
self.direction = self.LEFT
|
||||
self.character = '<'
|
||||
elif keystroke.name == "KEY_DOWN":
|
||||
self.direction = self.DOWN
|
||||
self.character = 'v'
|
||||
|
||||
def can_move(self, position, game):
|
||||
return game.on_board(position) and (game.is_empty(position) or self.is_on_apple(position, game))
|
||||
|
||||
def is_on_apple(self, position, game):
|
||||
return game.get_agent_by_name("Apple").position == position
|
||||
|
||||
|
||||
def create_game():
|
||||
head = SnakeHead()
|
||||
apple = Apple()
|
||||
game = Game(
|
||||
[head, apple],
|
||||
{'score': 0, 'reward': 0, 'energy': 150},
|
||||
board_size=(32, 16),
|
||||
framerate=12,
|
||||
show_state=['score'],
|
||||
)
|
||||
bw, bh = game.board_size
|
||||
head.position = (randint(1, bw - 2), randint(1, bh - 2))
|
||||
direction, character = choice([
|
||||
(SnakeHead.RIGHT, '>'),
|
||||
(SnakeHead.UP, '^'),
|
||||
(SnakeHead.LEFT, '<'),
|
||||
(SnakeHead.DOWN, 'v'),
|
||||
])
|
||||
head.direction = direction
|
||||
head.character = character
|
||||
apple.relocate(game)
|
||||
return game
|
||||
10
games/snake_v1/pyproject.toml
Normal file
10
games/snake_v1/pyproject.toml
Normal file
@@ -0,0 +1,10 @@
|
||||
[project]
|
||||
name = "snake_v1"
|
||||
version = "0.1.0"
|
||||
description = "Snake (v1): full board, no extra features"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = ["retro-games>=2.5.0"]
|
||||
|
||||
[tool.retro-gamer]
|
||||
actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"]
|
||||
reward = "reward"
|
||||
@@ -4,8 +4,8 @@ version = "0.1.0"
|
||||
description = "Reinforcement learning lab"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"retro-games>=2.4.0",
|
||||
"retro-gamer>=0.2.0",
|
||||
"retro-games>=2.5.0",
|
||||
"retro-gamer>=0.3.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
@@ -13,4 +13,7 @@ requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["babysnake", "forager"]
|
||||
packages = ["games", "q_learning"]
|
||||
|
||||
[tool.uv.sources]
|
||||
retro-gamer = { path = "../../packages/retro-gamer", editable = true }
|
||||
|
||||
37
questions.md
37
questions.md
@@ -26,3 +26,40 @@
|
||||
read the policy? For a given state, does the highest Q-value point toward the food?
|
||||
|
||||
8. How does the trained agent's behavior compare to the reasoning you wrote down in question 1?
|
||||
|
||||
## Checkpoint 4
|
||||
|
||||
9. **Full board (runs/snake-v1, ep_5000):** Describe the agent's behavior. Does it seem to know
|
||||
where the apple is? Does it move randomly or with some purpose?
|
||||
|
||||
10. **Features only (runs/snake-v2, ep_3000):** How does this agent differ from the v1 agent?
|
||||
What is it doing better? What is it doing that leads to shorter episodes?
|
||||
|
||||
11. **Final run, early (runs/snake, ep_1300):** This agent uses the egocentric view plus
|
||||
apple_dx/apple_dy. What has it learned that neither v1 nor v2 showed?
|
||||
|
||||
12. **Final run, mature (runs/snake, ep_20000):** What does this agent do well? Where does it
|
||||
still make mistakes?
|
||||
|
||||
13. In the features-only run (v2), reward rose as episodes got shorter. Why does a snake agent
|
||||
that is getting better at finding apples end up with shorter episodes?
|
||||
|
||||
14. The egocentric view crops the observation to a 17×17 window centered on the snake's head.
|
||||
What did the agent gain from this change, and what information did it lose access to?
|
||||
|
||||
## Checkpoint 5
|
||||
|
||||
Answer these questions after completing both training experiments in "Training Frogger."
|
||||
|
||||
15. **Hypothesis (Attempt 1):** Before training, predict what will happen. Will the agent learn to
|
||||
reach the top of the board? What challenge do you think it will face?
|
||||
|
||||
16. **Evidence (Attempt 1):** Copy the first three and last three lines of `runs/frogger/training.log`.
|
||||
Did training go as expected?
|
||||
|
||||
17. **Analysis (Attempt 1):** What did the agent learn to do? Where did it struggle?
|
||||
|
||||
18. **Experiment (Attempt 2):** What one thing did you change? Write your prediction, show the
|
||||
evidence (first and last few log lines), and describe what happened.
|
||||
|
||||
19. Which attempt produced the best agent? What would you try next if you had more time?
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,53 +0,0 @@
|
||||
[game]
|
||||
module = "retro.examples.snake"
|
||||
|
||||
[metadata]
|
||||
actions = [
|
||||
"KEY_RIGHT",
|
||||
"KEY_UP",
|
||||
"KEY_LEFT",
|
||||
"KEY_DOWN",
|
||||
]
|
||||
reward = "reward"
|
||||
extras_size = 2
|
||||
board_size = [
|
||||
17,
|
||||
17,
|
||||
]
|
||||
character_set = [
|
||||
"@",
|
||||
"*",
|
||||
">",
|
||||
"<",
|
||||
"^",
|
||||
"v",
|
||||
]
|
||||
observation_function = "snake_observation:egocentric_observation"
|
||||
|
||||
[preprocessing]
|
||||
spatial = false
|
||||
board = true
|
||||
observe_state = []
|
||||
|
||||
[model]
|
||||
hidden_sizes = [
|
||||
128,
|
||||
64,
|
||||
]
|
||||
|
||||
[training]
|
||||
learning_rate = 0.0001
|
||||
learning_rate_decay = 0.9999
|
||||
gamma = 0.99
|
||||
epsilon = 1.0
|
||||
epsilon_decay = 0.9997
|
||||
epsilon_min = 0.05
|
||||
batch_size = 64
|
||||
memory_capacity = 50000
|
||||
target_update_freq = 500
|
||||
train_every = 4
|
||||
training_episodes = 20000
|
||||
prioritize_experiences = true
|
||||
exploration_turns = 200
|
||||
unknown_character_strategy = "ignore"
|
||||
max_turns_per_episode = 2000
|
||||
@@ -1,213 +0,0 @@
|
||||
[INIT] === Network Architecture ===
|
||||
[INIT] Board: 17×17, character set: 6 chars (one-hot per cell)
|
||||
[INIT] Observed state features: 2 | Actions (incl. no-op): 5
|
||||
[INIT] spatial=False → using MLP architecture
|
||||
[INIT] Rationale: the board encodes UI/status rather than a spatial scene;
|
||||
[INIT] a flat MLP over the full observation is sufficient.
|
||||
[INIT] MLP: 1736 → 128 → 64 → 5
|
||||
[INIT] Hidden layers: 2 | Layer sizes: [128, 64]
|
||||
[INIT] Output: 5 Q-values
|
||||
[INIT] Actions: ['KEY_RIGHT', 'KEY_UP', 'KEY_LEFT', 'KEY_DOWN'] + (no-op)
|
||||
[INIT] Device: mps
|
||||
|
||||
=== Training started | 2026-06-23 22:39:07 ===
|
||||
[ep_0100] ep=0001-0100 avg_reward=-7.8 avg_steps=49 epsilon=0.970 avg_loss=0.8 time=0m06s total=0m06s
|
||||
[ep_0200] ep=0101-0200 avg_reward=-5.7 avg_steps=51 epsilon=0.942 avg_loss=0.9 time=0m07s total=0m13s
|
||||
[ep_0300] ep=0201-0300 avg_reward=+0.1 avg_steps=49 epsilon=0.914 avg_loss=1.2 time=0m06s total=0m20s
|
||||
[ep_0400] ep=0301-0400 avg_reward=+7.9 avg_steps=70 epsilon=0.887 avg_loss=1.6 time=0m09s total=0m30s
|
||||
[ep_0500] ep=0401-0500 avg_reward=+12.8 avg_steps=57 epsilon=0.861 avg_loss=1.9 time=0m08s total=0m38s
|
||||
[ep_0600] ep=0501-0600 avg_reward=+20.5 avg_steps=57 epsilon=0.835 avg_loss=2.3 time=0m08s total=0m47s
|
||||
[ep_0700] ep=0601-0700 avg_reward=+23.7 avg_steps=62 epsilon=0.811 avg_loss=2.8 time=0m10s total=0m58s
|
||||
[ep_0800] ep=0701-0800 avg_reward=+21.2 avg_steps=56 epsilon=0.787 avg_loss=3.4 time=0m10s total=1m08s
|
||||
[ep_0900] ep=0801-0900 avg_reward=+30.1 avg_steps=55 epsilon=0.763 avg_loss=4.1 time=0m10s total=1m19s
|
||||
[ep_1000] ep=0901-1000 avg_reward=+33.6 avg_steps=59 epsilon=0.741 avg_loss=4.6 time=0m11s total=1m31s
|
||||
[ep_1100] ep=1001-1100 avg_reward=+36.0 avg_steps=58 epsilon=0.719 avg_loss=5.4 time=0m11s total=1m43s
|
||||
[ep_1200] ep=1101-1200 avg_reward=+34.0 avg_steps=50 epsilon=0.698 avg_loss=6.1 time=0m10s total=1m53s
|
||||
[ep_1300] ep=1201-1300 avg_reward=+31.6 avg_steps=49 epsilon=0.677 avg_loss=6.7 time=0m09s total=2m03s
|
||||
[ep_1400] ep=1301-1400 avg_reward=+30.4 avg_steps=60 epsilon=0.657 avg_loss=7.2 time=0m12s total=2m15s
|
||||
[ep_1500] ep=1401-1500 avg_reward=+28.0 avg_steps=65 epsilon=0.638 avg_loss=7.6 time=0m13s total=2m28s
|
||||
[ep_1600] ep=1501-1600 avg_reward=+16.3 avg_steps=68 epsilon=0.619 avg_loss=7.6 time=0m14s total=2m42s
|
||||
[ep_1700] ep=1601-1700 avg_reward=+10.1 avg_steps=76 epsilon=0.600 avg_loss=7.7 time=0m18s total=3m01s
|
||||
[ep_1800] ep=1701-1800 avg_reward=+6.9 avg_steps=89 epsilon=0.583 avg_loss=7.6 time=0m20s total=3m21s
|
||||
[ep_1900] ep=1801-1900 avg_reward=+1.9 avg_steps=85 epsilon=0.565 avg_loss=7.4 time=0m16s total=3m38s
|
||||
[ep_2000] ep=1901-2000 avg_reward=+1.1 avg_steps=93 epsilon=0.549 avg_loss=6.9 time=0m18s total=3m57s
|
||||
[ep_2100] ep=2001-2100 avg_reward=+1.4 avg_steps=105 epsilon=0.533 avg_loss=6.2 time=0m21s total=4m18s
|
||||
[ep_2200] ep=2101-2200 avg_reward=-0.8 avg_steps=83 epsilon=0.517 avg_loss=5.5 time=0m16s total=4m34s
|
||||
[ep_2300] ep=2201-2300 avg_reward=-3.8 avg_steps=87 epsilon=0.502 avg_loss=5.1 time=0m17s total=4m52s
|
||||
[ep_2400] ep=2301-2400 avg_reward=-0.1 avg_steps=89 epsilon=0.487 avg_loss=4.6 time=0m17s total=5m09s
|
||||
[ep_2500] ep=2401-2500 avg_reward=-2.5 avg_steps=94 epsilon=0.472 avg_loss=4.1 time=0m18s total=5m28s
|
||||
[ep_2600] ep=2501-2600 avg_reward=+1.2 avg_steps=97 epsilon=0.458 avg_loss=3.7 time=0m19s total=5m48s
|
||||
[ep_2700] ep=2601-2700 avg_reward=+8.0 avg_steps=85 epsilon=0.445 avg_loss=3.4 time=0m16s total=6m05s
|
||||
[ep_2800] ep=2701-2800 avg_reward=+5.2 avg_steps=83 epsilon=0.432 avg_loss=2.9 time=0m16s total=6m21s
|
||||
[ep_2900] ep=2801-2900 avg_reward=+8.4 avg_steps=89 epsilon=0.419 avg_loss=2.7 time=0m17s total=6m39s
|
||||
[ep_3000] ep=2901-3000 avg_reward=+4.0 avg_steps=98 epsilon=0.407 avg_loss=2.4 time=0m19s total=6m59s
|
||||
[ep_3100] ep=3001-3100 avg_reward=+6.0 avg_steps=80 epsilon=0.394 avg_loss=2.1 time=0m16s total=7m15s
|
||||
[ep_3200] ep=3101-3200 avg_reward=+7.0 avg_steps=86 epsilon=0.383 avg_loss=2.0 time=0m17s total=7m33s
|
||||
[ep_3300] ep=3201-3300 avg_reward=+4.3 avg_steps=93 epsilon=0.372 avg_loss=1.8 time=0m19s total=7m52s
|
||||
[ep_3400] ep=3301-3400 avg_reward=+2.7 avg_steps=98 epsilon=0.361 avg_loss=1.8 time=0m20s total=8m12s
|
||||
[ep_3500] ep=3401-3500 avg_reward=+7.7 avg_steps=99 epsilon=0.350 avg_loss=1.6 time=0m20s total=8m32s
|
||||
[ep_3600] ep=3501-3600 avg_reward=+21.0 avg_steps=86 epsilon=0.340 avg_loss=1.5 time=0m17s total=8m50s
|
||||
[ep_3700] ep=3601-3700 avg_reward=+30.8 avg_steps=67 epsilon=0.330 avg_loss=1.4 time=0m13s total=9m03s
|
||||
[ep_3800] ep=3701-3800 avg_reward=+38.4 avg_steps=53 epsilon=0.320 avg_loss=1.3 time=0m10s total=9m14s
|
||||
[ep_3900] ep=3801-3900 avg_reward=+48.0 avg_steps=49 epsilon=0.310 avg_loss=1.4 time=0m09s total=9m24s
|
||||
[ep_4000] ep=3901-4000 avg_reward=+53.2 avg_steps=42 epsilon=0.301 avg_loss=1.4 time=0m08s total=9m33s
|
||||
[ep_4100] ep=4001-4100 avg_reward=+49.1 avg_steps=47 epsilon=0.292 avg_loss=1.5 time=0m09s total=9m42s
|
||||
[ep_4200] ep=4101-4200 avg_reward=+49.9 avg_steps=50 epsilon=0.284 avg_loss=1.6 time=0m10s total=9m53s
|
||||
[ep_4300] ep=4201-4300 avg_reward=+54.5 avg_steps=48 epsilon=0.275 avg_loss=1.7 time=0m09s total=10m02s
|
||||
[ep_4400] ep=4301-4400 avg_reward=+57.5 avg_steps=48 epsilon=0.267 avg_loss=1.8 time=0m09s total=10m12s
|
||||
[ep_4500] ep=4401-4500 avg_reward=+53.6 avg_steps=47 epsilon=0.259 avg_loss=1.9 time=0m09s total=10m22s
|
||||
[ep_4600] ep=4501-4600 avg_reward=+59.3 avg_steps=44 epsilon=0.252 avg_loss=1.9 time=0m09s total=10m31s
|
||||
[ep_4700] ep=4601-4700 avg_reward=+60.2 avg_steps=39 epsilon=0.244 avg_loss=1.8 time=0m08s total=10m39s
|
||||
[ep_4800] ep=4701-4800 avg_reward=+58.6 avg_steps=40 epsilon=0.237 avg_loss=1.8 time=0m08s total=10m47s
|
||||
[ep_4900] ep=4801-4900 avg_reward=+64.5 avg_steps=54 epsilon=0.230 avg_loss=1.9 time=0m11s total=10m59s
|
||||
[ep_5000] ep=4901-5000 avg_reward=+72.9 avg_steps=37 epsilon=0.223 avg_loss=1.9 time=0m07s total=11m06s
|
||||
[ep_5100] ep=5001-5100 avg_reward=+72.2 avg_steps=36 epsilon=0.216 avg_loss=1.9 time=0m07s total=11m14s
|
||||
[ep_5200] ep=5101-5200 avg_reward=+71.6 avg_steps=39 epsilon=0.210 avg_loss=1.9 time=0m08s total=11m22s
|
||||
[ep_5300] ep=5201-5300 avg_reward=+67.9 avg_steps=34 epsilon=0.204 avg_loss=1.9 time=0m07s total=11m29s
|
||||
[ep_5400] ep=5301-5400 avg_reward=+89.2 avg_steps=43 epsilon=0.198 avg_loss=1.9 time=0m09s total=11m38s
|
||||
[ep_5500] ep=5401-5500 avg_reward=+92.0 avg_steps=42 epsilon=0.192 avg_loss=2.0 time=0m08s total=11m47s
|
||||
[ep_5600] ep=5501-5600 avg_reward=+86.4 avg_steps=35 epsilon=0.186 avg_loss=2.1 time=0m07s total=11m55s
|
||||
[ep_5700] ep=5601-5700 avg_reward=+92.5 avg_steps=37 epsilon=0.181 avg_loss=2.2 time=0m07s total=12m02s
|
||||
[ep_5800] ep=5701-5800 avg_reward=+98.5 avg_steps=45 epsilon=0.175 avg_loss=2.3 time=0m09s total=12m12s
|
||||
[ep_5900] ep=5801-5900 avg_reward=+100.9 avg_steps=39 epsilon=0.170 avg_loss=2.5 time=0m08s total=12m20s
|
||||
[ep_6000] ep=5901-6000 avg_reward=+95.3 avg_steps=40 epsilon=0.165 avg_loss=2.7 time=0m08s total=12m28s
|
||||
[ep_6100] ep=6001-6100 avg_reward=+92.4 avg_steps=40 epsilon=0.160 avg_loss=2.9 time=0m08s total=12m37s
|
||||
[ep_6200] ep=6101-6200 avg_reward=+100.2 avg_steps=45 epsilon=0.156 avg_loss=3.1 time=0m09s total=12m46s
|
||||
[ep_6300] ep=6201-6300 avg_reward=+96.4 avg_steps=43 epsilon=0.151 avg_loss=3.5 time=0m09s total=12m56s
|
||||
[ep_6400] ep=6301-6400 avg_reward=+107.5 avg_steps=45 epsilon=0.147 avg_loss=3.9 time=0m09s total=13m05s
|
||||
[ep_6500] ep=6401-6500 avg_reward=+87.4 avg_steps=38 epsilon=0.142 avg_loss=4.1 time=0m08s total=13m13s
|
||||
[ep_6600] ep=6501-6600 avg_reward=+126.1 avg_steps=51 epsilon=0.138 avg_loss=4.7 time=0m10s total=13m24s
|
||||
[ep_6700] ep=6601-6700 avg_reward=+118.6 avg_steps=42 epsilon=0.134 avg_loss=5.1 time=0m08s total=13m33s
|
||||
[ep_6800] ep=6701-6800 avg_reward=+117.7 avg_steps=44 epsilon=0.130 avg_loss=5.7 time=0m09s total=13m42s
|
||||
[ep_6900] ep=6801-6900 avg_reward=+154.6 avg_steps=56 epsilon=0.126 avg_loss=6.2 time=0m11s total=13m54s
|
||||
[ep_7000] ep=6901-7000 avg_reward=+129.3 avg_steps=44 epsilon=0.122 avg_loss=6.8 time=0m09s total=14m04s
|
||||
[ep_7100] ep=7001-7100 avg_reward=+114.2 avg_steps=40 epsilon=0.119 avg_loss=7.2 time=0m08s total=14m12s
|
||||
[ep_7200] ep=7101-7200 avg_reward=+147.2 avg_steps=52 epsilon=0.115 avg_loss=7.6 time=0m11s total=14m24s
|
||||
[ep_7300] ep=7201-7300 avg_reward=+122.9 avg_steps=45 epsilon=0.112 avg_loss=8.1 time=0m09s total=14m33s
|
||||
[ep_7400] ep=7301-7400 avg_reward=+136.2 avg_steps=47 epsilon=0.109 avg_loss=8.4 time=0m10s total=14m43s
|
||||
[ep_7500] ep=7401-7500 avg_reward=+151.2 avg_steps=51 epsilon=0.105 avg_loss=8.5 time=0m11s total=14m54s
|
||||
[ep_7600] ep=7501-7600 avg_reward=+150.5 avg_steps=56 epsilon=0.102 avg_loss=8.5 time=0m11s total=15m06s
|
||||
[ep_7700] ep=7601-7700 avg_reward=+120.3 avg_steps=46 epsilon=0.099 avg_loss=8.6 time=0m09s total=15m16s
|
||||
[ep_7800] ep=7701-7800 avg_reward=+121.5 avg_steps=47 epsilon=0.096 avg_loss=8.5 time=0m09s total=15m26s
|
||||
[ep_7900] ep=7801-7900 avg_reward=+160.6 avg_steps=60 epsilon=0.093 avg_loss=8.7 time=0m12s total=15m39s
|
||||
[ep_8000] ep=7901-8000 avg_reward=+134.2 avg_steps=53 epsilon=0.091 avg_loss=8.6 time=0m11s total=15m50s
|
||||
[ep_8100] ep=8001-8100 avg_reward=+141.0 avg_steps=51 epsilon=0.088 avg_loss=8.5 time=0m10s total=16m01s
|
||||
[ep_8200] ep=8101-8200 avg_reward=+167.7 avg_steps=57 epsilon=0.085 avg_loss=8.2 time=0m12s total=16m13s
|
||||
[ep_8300] ep=8201-8300 avg_reward=+165.9 avg_steps=56 epsilon=0.083 avg_loss=8.0 time=0m11s total=16m25s
|
||||
[ep_8400] ep=8301-8400 avg_reward=+199.9 avg_steps=71 epsilon=0.080 avg_loss=7.9 time=0m15s total=16m40s
|
||||
[ep_8500] ep=8401-8500 avg_reward=+196.7 avg_steps=66 epsilon=0.078 avg_loss=7.5 time=0m14s total=16m54s
|
||||
[ep_8600] ep=8501-8600 avg_reward=+218.4 avg_steps=71 epsilon=0.076 avg_loss=7.4 time=0m15s total=17m10s
|
||||
[ep_8700] ep=8601-8700 avg_reward=+204.1 avg_steps=74 epsilon=0.074 avg_loss=7.1 time=0m15s total=17m25s
|
||||
[ep_8800] ep=8701-8800 avg_reward=+216.2 avg_steps=70 epsilon=0.071 avg_loss=6.8 time=0m14s total=17m40s
|
||||
[ep_8900] ep=8801-8900 avg_reward=+167.1 avg_steps=58 epsilon=0.069 avg_loss=6.7 time=0m12s total=17m53s
|
||||
[ep_9000] ep=8901-9000 avg_reward=+188.4 avg_steps=58 epsilon=0.067 avg_loss=6.6 time=0m12s total=18m05s
|
||||
[ep_9100] ep=9001-9100 avg_reward=+225.5 avg_steps=70 epsilon=0.065 avg_loss=6.6 time=0m14s total=18m20s
|
||||
[ep_9200] ep=9101-9200 avg_reward=+260.5 avg_steps=81 epsilon=0.063 avg_loss=6.5 time=0m17s total=18m37s
|
||||
[ep_9300] ep=9201-9300 avg_reward=+250.0 avg_steps=77 epsilon=0.061 avg_loss=6.3 time=0m16s total=18m54s
|
||||
[ep_9400] ep=9301-9400 avg_reward=+313.0 avg_steps=93 epsilon=0.060 avg_loss=6.1 time=0m19s total=19m14s
|
||||
[ep_9500] ep=9401-9500 avg_reward=+314.2 avg_steps=93 epsilon=0.058 avg_loss=5.9 time=0m19s total=19m34s
|
||||
[ep_9600] ep=9501-9600 avg_reward=+246.5 avg_steps=75 epsilon=0.056 avg_loss=5.7 time=0m16s total=19m50s
|
||||
[ep_9700] ep=9601-9700 avg_reward=+274.8 avg_steps=82 epsilon=0.054 avg_loss=5.5 time=0m17s total=20m07s
|
||||
[ep_9800] ep=9701-9800 avg_reward=+329.7 avg_steps=97 epsilon=0.053 avg_loss=5.5 time=0m20s total=20m28s
|
||||
[ep_9900] ep=9801-9900 avg_reward=+286.5 avg_steps=87 epsilon=0.051 avg_loss=5.4 time=0m18s total=20m47s
|
||||
[ep_10000] ep=9901-10000 avg_reward=+304.0 avg_steps=88 epsilon=0.050 avg_loss=5.5 time=0m18s total=21m05s
|
||||
[ep_10100] ep=10001-10100 avg_reward=+349.1 avg_steps=99 epsilon=0.050 avg_loss=5.5 time=0m21s total=21m26s
|
||||
[ep_10200] ep=10101-10200 avg_reward=+327.1 avg_steps=93 epsilon=0.050 avg_loss=5.3 time=0m19s total=21m46s
|
||||
[ep_10300] ep=10201-10300 avg_reward=+347.8 avg_steps=98 epsilon=0.050 avg_loss=5.3 time=0m20s total=22m07s
|
||||
[ep_10400] ep=10301-10400 avg_reward=+327.0 avg_steps=96 epsilon=0.050 avg_loss=5.1 time=0m20s total=22m28s
|
||||
[ep_10500] ep=10401-10500 avg_reward=+330.2 avg_steps=96 epsilon=0.050 avg_loss=5.1 time=0m20s total=22m49s
|
||||
[ep_10600] ep=10501-10600 avg_reward=+335.0 avg_steps=92 epsilon=0.050 avg_loss=5.0 time=0m19s total=23m09s
|
||||
[ep_10700] ep=10601-10700 avg_reward=+313.9 avg_steps=88 epsilon=0.050 avg_loss=5.0 time=0m19s total=23m28s
|
||||
[ep_10800] ep=10701-10800 avg_reward=+368.5 avg_steps=104 epsilon=0.050 avg_loss=5.0 time=0m22s total=23m50s
|
||||
[ep_10900] ep=10801-10900 avg_reward=+366.0 avg_steps=104 epsilon=0.050 avg_loss=4.9 time=0m22s total=24m12s
|
||||
[ep_11000] ep=10901-11000 avg_reward=+309.2 avg_steps=86 epsilon=0.050 avg_loss=4.9 time=0m18s total=24m31s
|
||||
[ep_11100] ep=11001-11100 avg_reward=+354.0 avg_steps=100 epsilon=0.050 avg_loss=4.9 time=0m21s total=24m53s
|
||||
[ep_11200] ep=11101-11200 avg_reward=+305.9 avg_steps=84 epsilon=0.050 avg_loss=4.9 time=0m18s total=25m11s
|
||||
[ep_11300] ep=11201-11300 avg_reward=+304.6 avg_steps=87 epsilon=0.050 avg_loss=4.9 time=0m18s total=25m30s
|
||||
[ep_11400] ep=11301-11400 avg_reward=+368.9 avg_steps=101 epsilon=0.050 avg_loss=4.8 time=0m21s total=25m52s
|
||||
[ep_11500] ep=11401-11500 avg_reward=+342.0 avg_steps=93 epsilon=0.050 avg_loss=4.7 time=0m20s total=26m12s
|
||||
[ep_11600] ep=11501-11600 avg_reward=+304.7 avg_steps=86 epsilon=0.050 avg_loss=4.7 time=0m18s total=26m30s
|
||||
[ep_11700] ep=11601-11700 avg_reward=+354.6 avg_steps=99 epsilon=0.050 avg_loss=4.7 time=0m21s total=26m52s
|
||||
[ep_11800] ep=11701-11800 avg_reward=+308.3 avg_steps=89 epsilon=0.050 avg_loss=4.7 time=0m19s total=27m11s
|
||||
[ep_11900] ep=11801-11900 avg_reward=+321.1 avg_steps=93 epsilon=0.050 avg_loss=4.6 time=0m20s total=27m31s
|
||||
[ep_12000] ep=11901-12000 avg_reward=+334.3 avg_steps=97 epsilon=0.050 avg_loss=4.7 time=0m20s total=27m52s
|
||||
[ep_12100] ep=12001-12100 avg_reward=+367.0 avg_steps=104 epsilon=0.050 avg_loss=4.6 time=0m22s total=28m15s
|
||||
[ep_12200] ep=12101-12200 avg_reward=+346.9 avg_steps=103 epsilon=0.050 avg_loss=4.4 time=0m22s total=28m37s
|
||||
[ep_12300] ep=12201-12300 avg_reward=+333.0 avg_steps=96 epsilon=0.050 avg_loss=4.5 time=0m20s total=28m58s
|
||||
[ep_12400] ep=12301-12400 avg_reward=+341.2 avg_steps=98 epsilon=0.050 avg_loss=4.3 time=0m21s total=29m19s
|
||||
[ep_12500] ep=12401-12500 avg_reward=+342.6 avg_steps=97 epsilon=0.050 avg_loss=4.4 time=0m21s total=29m40s
|
||||
[ep_12600] ep=12501-12600 avg_reward=+334.0 avg_steps=92 epsilon=0.050 avg_loss=4.3 time=0m19s total=30m00s
|
||||
[ep_12700] ep=12601-12700 avg_reward=+332.6 avg_steps=92 epsilon=0.050 avg_loss=4.4 time=0m20s total=30m20s
|
||||
[ep_12800] ep=12701-12800 avg_reward=+343.0 avg_steps=96 epsilon=0.050 avg_loss=4.4 time=0m20s total=30m41s
|
||||
[ep_12900] ep=12801-12900 avg_reward=+344.9 avg_steps=101 epsilon=0.050 avg_loss=4.5 time=0m21s total=31m02s
|
||||
[ep_13000] ep=12901-13000 avg_reward=+292.4 avg_steps=83 epsilon=0.050 avg_loss=4.5 time=0m17s total=31m20s
|
||||
[ep_13100] ep=13001-13100 avg_reward=+323.1 avg_steps=89 epsilon=0.050 avg_loss=4.6 time=0m19s total=31m39s
|
||||
[ep_13200] ep=13101-13200 avg_reward=+310.8 avg_steps=86 epsilon=0.050 avg_loss=4.4 time=0m18s total=31m58s
|
||||
[ep_13300] ep=13201-13300 avg_reward=+308.8 avg_steps=85 epsilon=0.050 avg_loss=4.4 time=0m18s total=32m16s
|
||||
[ep_13400] ep=13301-13400 avg_reward=+395.9 avg_steps=111 epsilon=0.050 avg_loss=4.4 time=0m23s total=32m40s
|
||||
[ep_13500] ep=13401-13500 avg_reward=+395.5 avg_steps=111 epsilon=0.050 avg_loss=4.3 time=0m23s total=33m03s
|
||||
[ep_13600] ep=13501-13600 avg_reward=+384.9 avg_steps=107 epsilon=0.050 avg_loss=4.2 time=0m23s total=33m26s
|
||||
[ep_13700] ep=13601-13700 avg_reward=+342.1 avg_steps=96 epsilon=0.050 avg_loss=4.3 time=0m20s total=33m47s
|
||||
[ep_13800] ep=13701-13800 avg_reward=+319.3 avg_steps=92 epsilon=0.050 avg_loss=4.3 time=0m21s total=34m09s
|
||||
[ep_13900] ep=13801-13900 avg_reward=+337.6 avg_steps=98 epsilon=0.050 avg_loss=4.2 time=0m25s total=34m35s
|
||||
[ep_14000] ep=13901-14000 avg_reward=+302.8 avg_steps=85 epsilon=0.050 avg_loss=4.3 time=0m18s total=34m53s
|
||||
[ep_14100] ep=14001-14100 avg_reward=+337.7 avg_steps=94 epsilon=0.050 avg_loss=4.4 time=0m20s total=35m13s
|
||||
[ep_14200] ep=14101-14200 avg_reward=+315.7 avg_steps=88 epsilon=0.050 avg_loss=4.2 time=0m18s total=35m32s
|
||||
[ep_14300] ep=14201-14300 avg_reward=+337.8 avg_steps=99 epsilon=0.050 avg_loss=4.3 time=0m21s total=35m53s
|
||||
[ep_14400] ep=14301-14400 avg_reward=+313.3 avg_steps=88 epsilon=0.050 avg_loss=4.3 time=0m18s total=36m12s
|
||||
[ep_14500] ep=14401-14500 avg_reward=+322.5 avg_steps=91 epsilon=0.050 avg_loss=4.3 time=0m19s total=36m32s
|
||||
[ep_14600] ep=14501-14600 avg_reward=+385.1 avg_steps=106 epsilon=0.050 avg_loss=4.2 time=0m22s total=36m55s
|
||||
[ep_14700] ep=14601-14700 avg_reward=+333.2 avg_steps=92 epsilon=0.050 avg_loss=4.1 time=0m19s total=37m14s
|
||||
[ep_14800] ep=14701-14800 avg_reward=+325.7 avg_steps=92 epsilon=0.050 avg_loss=4.0 time=0m19s total=37m34s
|
||||
[ep_14900] ep=14801-14900 avg_reward=+308.1 avg_steps=87 epsilon=0.050 avg_loss=4.2 time=0m18s total=37m53s
|
||||
[ep_15000] ep=14901-15000 avg_reward=+374.5 avg_steps=103 epsilon=0.050 avg_loss=4.2 time=0m22s total=38m15s
|
||||
[ep_15100] ep=15001-15100 avg_reward=+334.0 avg_steps=94 epsilon=0.050 avg_loss=4.3 time=0m20s total=38m35s
|
||||
[ep_15200] ep=15101-15200 avg_reward=+363.1 avg_steps=97 epsilon=0.050 avg_loss=4.4 time=0m20s total=38m56s
|
||||
[ep_15300] ep=15201-15300 avg_reward=+367.9 avg_steps=104 epsilon=0.050 avg_loss=4.5 time=0m22s total=39m18s
|
||||
[ep_15400] ep=15301-15400 avg_reward=+367.8 avg_steps=99 epsilon=0.050 avg_loss=4.4 time=0m21s total=39m39s
|
||||
[ep_15500] ep=15401-15500 avg_reward=+331.6 avg_steps=93 epsilon=0.050 avg_loss=4.2 time=0m19s total=39m59s
|
||||
[ep_15600] ep=15501-15600 avg_reward=+349.4 avg_steps=97 epsilon=0.050 avg_loss=4.1 time=0m20s total=40m20s
|
||||
[ep_15700] ep=15601-15700 avg_reward=+374.1 avg_steps=108 epsilon=0.050 avg_loss=4.1 time=0m23s total=40m43s
|
||||
[ep_15800] ep=15701-15800 avg_reward=+345.4 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=41m04s
|
||||
[ep_15900] ep=15801-15900 avg_reward=+372.3 avg_steps=108 epsilon=0.050 avg_loss=4.1 time=0m23s total=41m27s
|
||||
[ep_16000] ep=15901-16000 avg_reward=+351.0 avg_steps=95 epsilon=0.050 avg_loss=4.2 time=0m20s total=41m47s
|
||||
[ep_16100] ep=16001-16100 avg_reward=+345.8 avg_steps=99 epsilon=0.050 avg_loss=4.2 time=0m21s total=42m08s
|
||||
[ep_16200] ep=16101-16200 avg_reward=+403.9 avg_steps=110 epsilon=0.050 avg_loss=4.2 time=0m23s total=42m32s
|
||||
[ep_16300] ep=16201-16300 avg_reward=+424.1 avg_steps=116 epsilon=0.050 avg_loss=4.2 time=0m24s total=42m57s
|
||||
[ep_16400] ep=16301-16400 avg_reward=+383.7 avg_steps=104 epsilon=0.050 avg_loss=4.3 time=0m22s total=43m19s
|
||||
[ep_16500] ep=16401-16500 avg_reward=+356.4 avg_steps=97 epsilon=0.050 avg_loss=4.1 time=0m20s total=43m40s
|
||||
[ep_16600] ep=16501-16600 avg_reward=+367.3 avg_steps=104 epsilon=0.050 avg_loss=4.1 time=0m22s total=44m02s
|
||||
[ep_16700] ep=16601-16700 avg_reward=+322.2 avg_steps=96 epsilon=0.050 avg_loss=4.0 time=0m20s total=44m23s
|
||||
[ep_16800] ep=16701-16800 avg_reward=+376.6 avg_steps=103 epsilon=0.050 avg_loss=4.1 time=0m22s total=44m45s
|
||||
[ep_16900] ep=16801-16900 avg_reward=+340.6 avg_steps=96 epsilon=0.050 avg_loss=4.1 time=0m20s total=45m06s
|
||||
[ep_17000] ep=16901-17000 avg_reward=+353.4 avg_steps=99 epsilon=0.050 avg_loss=4.0 time=0m21s total=45m27s
|
||||
[ep_17100] ep=17001-17100 avg_reward=+304.6 avg_steps=87 epsilon=0.050 avg_loss=4.0 time=0m18s total=45m46s
|
||||
[ep_17200] ep=17101-17200 avg_reward=+322.3 avg_steps=90 epsilon=0.050 avg_loss=3.9 time=0m19s total=46m05s
|
||||
[ep_17300] ep=17201-17300 avg_reward=+376.1 avg_steps=105 epsilon=0.050 avg_loss=4.0 time=0m22s total=46m28s
|
||||
[ep_17400] ep=17301-17400 avg_reward=+333.3 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=46m48s
|
||||
[ep_17500] ep=17401-17500 avg_reward=+363.4 avg_steps=104 epsilon=0.050 avg_loss=4.0 time=0m22s total=47m10s
|
||||
[ep_17600] ep=17501-17600 avg_reward=+336.3 avg_steps=93 epsilon=0.050 avg_loss=3.9 time=0m20s total=47m30s
|
||||
[ep_17700] ep=17601-17700 avg_reward=+347.2 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=47m51s
|
||||
[ep_17800] ep=17701-17800 avg_reward=+344.4 avg_steps=98 epsilon=0.050 avg_loss=4.0 time=0m21s total=48m12s
|
||||
[ep_17900] ep=17801-17900 avg_reward=+371.6 avg_steps=109 epsilon=0.050 avg_loss=4.0 time=0m23s total=48m35s
|
||||
[ep_18000] ep=17901-18000 avg_reward=+433.1 avg_steps=119 epsilon=0.050 avg_loss=4.1 time=0m25s total=49m00s
|
||||
[ep_18100] ep=18001-18100 avg_reward=+317.4 avg_steps=90 epsilon=0.050 avg_loss=4.2 time=0m19s total=49m20s
|
||||
[ep_18200] ep=18101-18200 avg_reward=+381.8 avg_steps=106 epsilon=0.050 avg_loss=4.2 time=0m22s total=49m42s
|
||||
[ep_18300] ep=18201-18300 avg_reward=+377.0 avg_steps=105 epsilon=0.050 avg_loss=4.1 time=0m22s total=50m05s
|
||||
[ep_18400] ep=18301-18400 avg_reward=+377.0 avg_steps=101 epsilon=0.050 avg_loss=4.0 time=0m21s total=50m26s
|
||||
[ep_18500] ep=18401-18500 avg_reward=+356.1 avg_steps=101 epsilon=0.050 avg_loss=4.0 time=0m21s total=50m48s
|
||||
[ep_18600] ep=18501-18600 avg_reward=+409.6 avg_steps=109 epsilon=0.050 avg_loss=4.1 time=0m23s total=51m11s
|
||||
[ep_18700] ep=18601-18700 avg_reward=+338.2 avg_steps=94 epsilon=0.050 avg_loss=4.1 time=0m20s total=51m31s
|
||||
[ep_18800] ep=18701-18800 avg_reward=+354.6 avg_steps=104 epsilon=0.050 avg_loss=4.2 time=0m22s total=51m53s
|
||||
[ep_18900] ep=18801-18900 avg_reward=+333.9 avg_steps=94 epsilon=0.050 avg_loss=4.2 time=0m20s total=52m14s
|
||||
[ep_19000] ep=18901-19000 avg_reward=+388.2 avg_steps=106 epsilon=0.050 avg_loss=4.4 time=0m22s total=52m36s
|
||||
[ep_19100] ep=19001-19100 avg_reward=+368.2 avg_steps=100 epsilon=0.050 avg_loss=4.3 time=0m21s total=52m58s
|
||||
[ep_19200] ep=19101-19200 avg_reward=+435.8 avg_steps=121 epsilon=0.050 avg_loss=4.2 time=0m25s total=53m24s
|
||||
[ep_19300] ep=19201-19300 avg_reward=+345.8 avg_steps=99 epsilon=0.050 avg_loss=4.1 time=0m21s total=53m45s
|
||||
[ep_19400] ep=19301-19400 avg_reward=+372.5 avg_steps=104 epsilon=0.050 avg_loss=4.1 time=0m22s total=54m07s
|
||||
[ep_19500] ep=19401-19500 avg_reward=+374.1 avg_steps=103 epsilon=0.050 avg_loss=3.9 time=0m21s total=54m29s
|
||||
[ep_19600] ep=19501-19600 avg_reward=+387.1 avg_steps=109 epsilon=0.050 avg_loss=3.9 time=0m23s total=54m53s
|
||||
[ep_19700] ep=19601-19700 avg_reward=+360.1 avg_steps=99 epsilon=0.050 avg_loss=3.9 time=0m21s total=55m14s
|
||||
[ep_19800] ep=19701-19800 avg_reward=+387.3 avg_steps=107 epsilon=0.050 avg_loss=4.0 time=0m22s total=55m37s
|
||||
[ep_19900] ep=19801-19900 avg_reward=+363.4 avg_steps=98 epsilon=0.050 avg_loss=3.9 time=0m20s total=55m58s
|
||||
[ep_20000] ep=19901-20000 avg_reward=+335.6 avg_steps=94 epsilon=0.050 avg_loss=4.0 time=0m19s total=56m17s
|
||||
@@ -1,31 +0,0 @@
|
||||
"""A custom observation_function for retro.examples.snake.
|
||||
|
||||
Crops the board to a window centered on the snake's head before encoding,
|
||||
using egocentric_board()/encode_board() from retro_gamer.observation as
|
||||
building blocks. This replaces retro_gamer's old built-in egocentric flags —
|
||||
cropping is now just something an observation_function does for itself.
|
||||
|
||||
Point a run's config.toml at this with:
|
||||
|
||||
[metadata]
|
||||
observation_function = "snake_observation:egocentric_observation"
|
||||
board_size = [17, 17] # must match RADIUS below: 2*8+1 = 17
|
||||
"""
|
||||
import numpy as np
|
||||
from retro.views.headless import HeadlessView
|
||||
from retro_gamer.observation import egocentric_board, encode_board, encode_state
|
||||
|
||||
CHARACTER_SET = ["@", "*", ">", "<", "^", "v"]
|
||||
RADIUS = 8
|
||||
|
||||
|
||||
def egocentric_observation(game):
|
||||
"""Board cropped to a (2*RADIUS+1)^2 window around the snake's head, plus apple_dx/apple_dy."""
|
||||
view = HeadlessView()
|
||||
view.on_game_start(game)
|
||||
view.render(game)
|
||||
head = game.get_agent_by_name("Snake head")
|
||||
cropped = egocentric_board(view.board_characters, head.position, RADIUS)
|
||||
board_vec = encode_board(cropped, CHARACTER_SET).flatten()
|
||||
extras = encode_state(game.state, ["apple_dx", "apple_dy"])
|
||||
return np.concatenate([board_vec, extras])
|
||||
@@ -1,144 +0,0 @@
|
||||
# Snake Training: Conceptual Questions
|
||||
|
||||
Answer each question in the space provided. Use evidence from the training log
|
||||
and your observations of the agent at different checkpoints to support your answers.
|
||||
|
||||
---
|
||||
|
||||
## 1. Feature selection
|
||||
|
||||
In the first training attempt, the agent received the full 32×16 game board as
|
||||
its input (6 × 32 × 16 = 3,072 numbers). The agent could see every character on
|
||||
the board, yet it never learned to reliably find the apple after 45,000 episodes.
|
||||
|
||||
When we added `apple_dx` and `apple_dy` — two numbers that encode the direction
|
||||
from the snake's head to the apple — performance improved dramatically within
|
||||
hundreds of episodes.
|
||||
|
||||
**Why didn't the board encoding help the agent find the apple? What did the two
|
||||
new features provide that the board encoding could not?**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
---
|
||||
|
||||
## 2. Dimensionality reduction
|
||||
|
||||
In the full-board experiment, the agent processed 3,072 input values. When we
|
||||
switched to the egocentric view (a 17×17 window centered on the snake's head),
|
||||
the board input shrank to 17 × 17 × 6 = 1,734 values.
|
||||
|
||||
**How many input values did the egocentric view save compared to the full board?
|
||||
What is one thing the agent gained from this change, and one thing it lost?**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
---
|
||||
|
||||
## 3. Exploration vs. exploitation
|
||||
|
||||
With `epsilon_decay = 0.995`, epsilon falls from 1.0 to 0.05 by episode ~450.
|
||||
With `epsilon_decay = 0.9997` (used in the final run), epsilon is still 0.55 at
|
||||
episode 2,000.
|
||||
|
||||
**Sketch a rough curve of epsilon over time for each setting. Why does slower
|
||||
decay produce a better-trained agent, even though it means the agent takes more
|
||||
random actions overall?**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
---
|
||||
|
||||
## 4. Runaway loss
|
||||
|
||||
In one intermediate experiment, the loss grew from around 35 to hundreds of
|
||||
thousands within a few hundred episodes:
|
||||
|
||||
```
|
||||
[ep_0300] avg_loss=48.7 avg_reward=+8.1
|
||||
[ep_0500] avg_loss=347 avg_reward=+12.4
|
||||
[ep_0700] avg_loss=4,102 avg_reward=+6.5
|
||||
[ep_1100] avg_loss=686,000 avg_reward=-3.1
|
||||
```
|
||||
|
||||
This happened because the learning algorithm was using MSE (mean squared error)
|
||||
loss, which is *quadratic* — an error of size 2 produces a loss of 4, an error
|
||||
of size 10 produces a loss of 100.
|
||||
|
||||
**Describe the feedback loop that caused the loss to spiral upward. Why does
|
||||
Huber loss (which is linear for large errors) break this cycle?**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
---
|
||||
|
||||
## 5. Interpreting the training curve
|
||||
|
||||
Look at `runs/snake/training.log`. The reward climbs, then dips, then climbs
|
||||
again:
|
||||
|
||||
```
|
||||
[ep_1300] avg_reward= +31.6 avg_steps=49
|
||||
[ep_2300] avg_reward= -3.8 avg_steps=87
|
||||
[ep_4000] avg_reward= +53.2 avg_steps=42
|
||||
[ep_20000] avg_reward=+335.6 avg_steps=94
|
||||
```
|
||||
|
||||
Notice that around episode 4,000, avg_steps dropped sharply (from ~87 to 42)
|
||||
at the same time reward jumped. Then by episode 20,000, steps rose again while
|
||||
reward kept climbing.
|
||||
|
||||
**What do you think the agent was doing at each of these stages? Use the
|
||||
avg_steps and avg_reward numbers to support your interpretation.**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
---
|
||||
|
||||
## 6. Policy observation
|
||||
|
||||
Run these commands to watch the agent at three checkpoints:
|
||||
|
||||
```
|
||||
retro-gamer play runs/snake --checkpoint ep_1300
|
||||
retro-gamer play runs/snake --checkpoint ep_4000
|
||||
retro-gamer play runs/snake --checkpoint ep_20000
|
||||
```
|
||||
|
||||
**Describe the agent's behavior at each checkpoint. What has the agent learned
|
||||
by episode 4,000 that it hadn't yet learned at episode 1,300? What does the
|
||||
episode 20,000 agent do that the earlier agents do not?**
|
||||
|
||||
*ep_1300:*
|
||||
|
||||
*ep_4000:*
|
||||
|
||||
*ep_20000:*
|
||||
|
||||
---
|
||||
|
||||
## 7. CNN vs. MLP
|
||||
|
||||
In the first attempt (full board, no explicit features), we used a CNN
|
||||
(`spatial = true`). In the final run (egocentric board + explicit features), we
|
||||
used an MLP (`spatial = false`).
|
||||
|
||||
**Why might an MLP be a reasonable choice when using the egocentric view, even
|
||||
though the input is still a 2D board? What does the CNN offer that the MLP does
|
||||
not, and why is that less important with an egocentric observation?**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
---
|
||||
|
||||
## 8. Hyperparameter comparison
|
||||
|
||||
Suppose you ran two otherwise identical training experiments:
|
||||
- Run A: `learning_rate = 0.001`
|
||||
- Run B: `learning_rate = 0.0001`
|
||||
|
||||
**Based on what you learned from the runaway loss in Question 4, predict what
|
||||
would happen in each run. What does this tell you about the trade-off when
|
||||
choosing a learning rate?**
|
||||
|
||||
*Your answer:*
|
||||
103
training_log.md
103
training_log.md
@@ -1,103 +0,0 @@
|
||||
# Forager Training Log
|
||||
|
||||
Document each training attempt below. For each attempt, write your hypothesis
|
||||
before you run the experiment, then fill in the evidence and analysis after.
|
||||
|
||||
Use `retro-gamer info runs/forager/` to see a summary of your run,
|
||||
`cat runs/forager/training.log` to see the full log, and
|
||||
`retro-gamer plot runs/forager/ -o runs/forager/training.png` to graph it.
|
||||
|
||||
---
|
||||
|
||||
## Attempt 1
|
||||
|
||||
### Hypothesis
|
||||
|
||||
*Before training, predict what will happen with the default configuration.
|
||||
Will the agent learn to find the food? How quickly? What might go wrong?*
|
||||
|
||||
Your prediction:
|
||||
|
||||
### Configuration
|
||||
|
||||
*Copy the relevant sections of `runs/forager/config.toml` here.*
|
||||
|
||||
```toml
|
||||
|
||||
```
|
||||
|
||||
### Evidence
|
||||
|
||||
*Paste the first and last few lines of your training log, and any interesting
|
||||
moments in between.*
|
||||
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
### Analysis
|
||||
|
||||
*What happened? How do the numbers — avg_reward, avg_steps, epsilon, avg_loss —
|
||||
tell the story of what the agent learned? Did the result match your prediction?*
|
||||
|
||||
---
|
||||
|
||||
## Attempt 2
|
||||
|
||||
### Hypothesis
|
||||
|
||||
*Based on what you observed in Attempt 1, what will you change and why?
|
||||
Predict the outcome.*
|
||||
|
||||
### Configuration
|
||||
|
||||
```toml
|
||||
|
||||
```
|
||||
|
||||
### Evidence
|
||||
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
### Analysis
|
||||
|
||||
---
|
||||
|
||||
## Attempt 3 (if needed)
|
||||
|
||||
### Hypothesis
|
||||
|
||||
### Configuration
|
||||
|
||||
```toml
|
||||
|
||||
```
|
||||
|
||||
### Evidence
|
||||
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
### Analysis
|
||||
|
||||
---
|
||||
|
||||
## Final analysis
|
||||
|
||||
**Which attempt produced the best-trained agent? Run `retro-gamer play` on your
|
||||
best run's checkpoints and describe what the agent does.**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
**Compare two of your attempts. What changed between them, and how did that
|
||||
change affect the training curve?**
|
||||
|
||||
*Your answer:*
|
||||
|
||||
**If you had more time, what would you try next to improve the agent further?
|
||||
Refer to specific hyperparameters or configuration options.**
|
||||
|
||||
*Your answer:*
|
||||
44
uv.lock
generated
44
uv.lock
generated
@@ -424,8 +424,8 @@ dependencies = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "retro-gamer", specifier = ">=0.2.0" },
|
||||
{ name = "retro-games", specifier = ">=2.4.0" },
|
||||
{ name = "retro-gamer", editable = "../../packages/retro-gamer" },
|
||||
{ name = "retro-games", specifier = ">=2.5.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1069,8 +1069,8 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "retro-gamer"
|
||||
version = "0.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
version = "0.3.4"
|
||||
source = { editable = "../../packages/retro-gamer" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
{ name = "matplotlib" },
|
||||
@@ -1083,21 +1083,41 @@ dependencies = [
|
||||
{ name = "torch" },
|
||||
{ name = "tqdm" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/04/22/a861bc238bb50a016bbe184ad909ffabffad605869f4ff829b17d32b8cd1/retro_gamer-0.2.0.tar.gz", hash = "sha256:44f3fee63aef6847b686fe4e32983c160012d59fa0563c94bdcdbb13d40c044d", size = 154128, upload-time = "2026-06-24T00:46:05.615Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/9e/bf7fb5e607cafcb2bca320c10f2670a6edb37bdc80358fbf671bf51636e3/retro_gamer-0.2.0-py3-none-any.whl", hash = "sha256:8fc91e6c8d7709268ac97e362182d40d4d440046e4b36604413c53039a2e4713", size = 27840, upload-time = "2026-06-24T00:46:04.535Z" },
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "click", specifier = ">=8.0" },
|
||||
{ name = "matplotlib", specifier = ">=3.7" },
|
||||
{ name = "numpy", specifier = ">=1.24" },
|
||||
{ name = "plotext", specifier = ">=5.0" },
|
||||
{ name = "retro-games", editable = "../../packages/retro" },
|
||||
{ name = "seaborn", specifier = ">=0.13" },
|
||||
{ name = "tomli-w", specifier = ">=1.0" },
|
||||
{ name = "torch", specifier = ">=2.0" },
|
||||
{ name = "tqdm", specifier = ">=4.0" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
documentation = [
|
||||
{ name = "sphinx", specifier = ">=7.0" },
|
||||
{ name = "sphinx-rtd-theme", specifier = ">=2.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "retro-games"
|
||||
version = "2.4.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
version = "2.5.0"
|
||||
source = { editable = "../../packages/retro" }
|
||||
dependencies = [
|
||||
{ name = "blessed" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/96/07/caafc47b5f40a0198467e0d994c6433b91c84c29cd235cf794dee4381ae7/retro_games-2.4.0.tar.gz", hash = "sha256:6a4d988afa922da69402b169b82e4136a18988f8d34b1740ea4f23382ab4ab35", size = 21723, upload-time = "2026-06-22T20:44:42.112Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/39/7c/e97207d25668790fd6a246be48beef798c8b41accd7c8c61026e3dfa9c62/retro_games-2.4.0-py3-none-any.whl", hash = "sha256:ea15765ea639d2284937dd0f32475c5ea6a9a9da1b54d09f35a2e8043efea2ba", size = 34830, upload-time = "2026-06-22T20:44:42.964Z" },
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [{ name = "blessed", specifier = ">=1.33.0" }]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
documentation = [
|
||||
{ name = "sphinx", specifier = ">=8.1.3" },
|
||||
{ name = "sphinx-rtd-theme", specifier = ">=3.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user