mwc/lab_reinforcement_learning
Template
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updates

Browse Source
This commit is contained in:
Chris Proctor
2026-06-26 20:59:25 -04:00
parent a9385f8296
commit 482f4f6cfa
5 changed files with 191 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
games/frogger/__init__.py
View File

@@ -1,19 +1,37 @@
"""Frogger: guide a frog across busy traffic lanes to reach the far side. """Frogger: guide a frog across busy traffic lanes to reach the far side.
The frog starts at the bottom row. Cars move across the lanes between the The frog starts at the bottom row and must cross to the top. Traffic lanes
start and goal. Each lane has cars moving at different speeds in alternating alternate direction; safe lanes (no cars) appear every few rows as rest zones.
directions. The frog earns +10 for each row advanced, +50 for reaching the The frog scores 100 each time it reaches the top and respawns at the bottom.
top row, and -10 for being hit by a car or falling off the edge. Episodes end It earns +1 reward for each upward step and +100 for crossing, but the game
when the frog reaches the top, gets hit, or energy runs out. ends immediately if a car hits it.
""" """
from random import randint
from retro.game import Game from retro.game import Game
BOARD_WIDTH = 20 BOARD_WIDTH = 20
BOARD_HEIGHT = 12 BOARD_HEIGHT = 12
NUM_LANES = BOARD_HEIGHT - 2 FRAMERATE = 8
START_ENERGY = 200 GAME_DURATION = 60 * FRAMERATE # ~60 seconds
# One entry per traffic row (rows 1–10). None marks a safe lane with no cars.
# Each traffic entry is (direction, speed, num_cars, car_length).
# direction: 1 = right, -1 = left
# speed: steps between moves (higher = slower)
# num_cars: how many cars in the lane
# car_length: how many cells each car occupies
LANES = [
( 1, 2, 2, 2), # row 1 fast
(-1, 4, 3, 2), # row 2 slow, crowded
( 1, 1, 1, 3), # row 3 very fast
None, # row 4 safe zone
(-1, 3, 2, 2), # row 5 medium
( 1, 2, 2, 3), # row 6 fast, long cars
(-1, 5, 3, 2), # row 7 slow, crowded
None, # row 8 safe zone
( 1, 1, 2, 2), # row 9 very fast
(-1, 3, 2, 3), # row 10 medium, long cars
]
class Frog: class Frog:
@@ -28,133 +46,141 @@ class Frog:
RIGHT = (1, 0) RIGHT = (1, 0)
def __init__(self): def __init__(self):
self._direction = self.UP self._pending_direction = None
self._frames_remaining = GAME_DURATION
def handle_keystroke(self, keystroke, game): def handle_keystroke(self, keystroke, game):
if keystroke.name == "KEY_UP": if keystroke.name == "KEY_UP":
self._direction = self.UP self._pending_direction = self.UP
elif keystroke.name == "KEY_DOWN": elif keystroke.name == "KEY_DOWN":
self._direction = self.DOWN self._pending_direction = self.DOWN
elif keystroke.name == "KEY_LEFT": elif keystroke.name == "KEY_LEFT":
self._direction = self.LEFT self._pending_direction = self.LEFT
elif keystroke.name == "KEY_RIGHT": elif keystroke.name == "KEY_RIGHT":
self._direction = self.RIGHT self._pending_direction = self.RIGHT
def play_turn(self, game): def play_turn(self, game):
self._frames_remaining -= 1
game.state['time_left'] = self._frames_remaining // FRAMERATE
if self._frames_remaining <= 0:
game.end()
return
if self._pending_direction is None:
return
dx, dy = self._pending_direction
self._pending_direction = None
bw, bh = game.board_size bw, bh = game.board_size
x, y = self.position x, y = self.position
dx, dy = self._direction
nx, ny = x + dx, y + dy nx, ny = x + dx, y + dy
if not (0 <= nx < bw): if not (0 <= nx < bw) or ny >= bh:
game.state['reward'] -= 10
game.state['energy'] -= 50
self._reset(game)
return
if not (0 <= ny < bh):
if ny < 0:
game.state['score'] += 50
game.state['reward'] += 50
else:
game.state['reward'] -= 5 game.state['reward'] -= 5
self._reset(game)
return return
prev_y = y if ny < 0:
self.position = (nx, ny) game.state['score'] += 100
game.state['energy'] -= 1 game.state['reward'] += 100
game.state['reward'] -= 0.01 self._respawn(game)
return
if ny < prev_y: self.position = (nx, ny)
advancement = prev_y - ny
game.state['score'] += advancement * 10 if ny < y:
game.state['reward'] += advancement * 5 game.state['reward'] += 1
for agent in game.agents: for agent in game.agents:
if hasattr(agent, '_is_car') and agent.position == self.position: if hasattr(agent, '_is_car') and agent.position == self.position:
game.state['reward'] -= 10 game.state['reward'] -= 10
game.state['energy'] -= 50 game.end()
self._reset(game)
return return
bw, bh = game.board_size def _respawn(self, game):
fx, fy = self.position
game.state['frog_x'] = fx / bw
game.state['frog_y'] = fy / bh
if game.state['energy'] <= 0:
game.end()
def _reset(self, game):
bw, bh = game.board_size bw, bh = game.board_size
self.position = (bw // 2, bh - 1) self.position = (bw // 2, bh - 1)
self._direction = self.UP self._pending_direction = None
if game.state['energy'] <= 0:
game.end()
class Car: class CarSegment:
"""One cell of a multi-cell car. Moved by the lead Car each turn."""
_is_car = True _is_car = True
character = 'X' character = 'X'
color = "red_on_black" color = "red_on_black"
def __init__(self, lane, speed, direction, start_x, board_width): def __init__(self, name, position):
self.name = f"Car {lane}_{start_x}" self.name = name
self.position = position
class Car:
"""Lead cell of a multi-cell car. Moves itself and all trailing segments."""
_is_car = True
character = 'X'
color = "red_on_black"
def __init__(self, name, lane, speed, direction, start_x, board_width, segments):
self.name = name
self._lane = lane self._lane = lane
self._speed = speed self._speed = speed
self._direction = direction self._direction = direction
self._board_width = board_width self._board_width = board_width
self._step = 0 self._step = 0
self.position = (start_x, lane) self.position = (start_x, lane)
self.segments = segments
def play_turn(self, game): def play_turn(self, game):
self._step += 1 self._step += 1
if self._step < self._speed: if self._step < self._speed:
return return
self._step = 0 self._step = 0
bw = self._board_width
x, y = self.position x, y = self.position
x = (x + self._direction) % self._board_width self.position = ((x + self._direction) % bw, y)
self.position = (x, y) for seg in self.segments:
sx, sy = seg.position
seg.position = ((sx + self._direction) % bw, sy)
frog = game.get_agent_by_name("Frog") frog = game.get_agent_by_name("Frog")
if frog.position == self.position: car_cells = {self.position} | {s.position for s in self.segments}
frog._reset(game) if frog.position in car_cells:
game.state['reward'] -= 10 game.state['reward'] -= 10
game.state['energy'] -= 50 game.end()
def create_game(): def create_game():
bw, bh = BOARD_WIDTH, BOARD_HEIGHT bw, bh = BOARD_WIDTH, BOARD_HEIGHT
frog = Frog() frog = Frog()
frog.position = (bw // 2, bh - 1) frog.position = (bw // 2, bh - 1)
agents = [frog] agents = [frog]
car_id = 0
for lane_idx, row in enumerate(range(1, bh - 1)): for lane_idx, row in enumerate(range(1, bh - 1)):
direction = 1 if lane_idx % 2 == 0 else -1 spec = LANES[lane_idx] if lane_idx < len(LANES) else None
speed = 2 + (lane_idx % 3) if spec is None:
num_cars = 2 + (lane_idx % 3) continue
direction, speed, num_cars, car_length = spec
spacing = bw // num_cars spacing = bw // num_cars
for i in range(num_cars): for i in range(num_cars):
start_x = (i * spacing + lane_idx * 3) % bw start_x = (i * spacing + lane_idx * 3) % bw
agents.append(Car(row, speed, direction, start_x, bw)) segments = []
car_id += 1 for j in range(1, car_length):
seg_x = (start_x - direction * j) % bw
seg = CarSegment(f"Car_{row}_{i}_seg{j}", (seg_x, row))
segments.append(seg)
agents.append(seg)
agents.append(Car(f"Car_{row}_{i}", row, speed, direction, start_x, bw, segments))
game = Game( return Game(
agents, agents,
{ {
'score': 0, 'score': 0,
'reward': 0.0, 'reward': 0.0,
'energy': START_ENERGY, 'time_left': GAME_DURATION // FRAMERATE,
'frog_x': (bw // 2) / bw,
'frog_y': (bh - 1) / bh,
}, },
board_size=(bw, bh), board_size=(bw, bh),
framerate=8, framerate=FRAMERATE,
show_state=['score', 'energy'], show_state=['score', 'time_left'],
) )
return game
if __name__ == '__main__': if __name__ == '__main__':

73
games/snake/observation.py
View File

@@ -2,9 +2,82 @@ import numpy as np
from retro.views.headless import HeadlessView from retro.views.headless import HeadlessView
from retro_gamer.observation import egocentric_board, encode_board, encode_state from retro_gamer.observation import egocentric_board, encode_board, encode_state
HEAD_CHARS = frozenset({">", "<", "^", "v"})
CHARACTER_SET = ["@", "*", ">", "<", "^", "v"] CHARACTER_SET = ["@", "*", ">", "<", "^", "v"]
CHARACTER_SET_NORMALIZED = ["@", "*", "H"]
RADIUS = 8 RADIUS = 8
DIRECTIONS = [(1, 0), (0, -1), (-1, 0), (0, 1)] # RIGHT, UP, LEFT, DOWN
RADIUS_WIDE = 8 # 17×17 egocentric window
RADIUS_NARROW = 3 # 7×7 egocentric window
def direction_observation(game):
"""Normalized 3-char board plus apple_dx, apple_dy, and a one-hot direction.
Returns board in channel-first (C, H, W) order for the CNN, followed by
apple_dx, apple_dy, and 4 one-hot bits encoding the snake's heading.
The board collapses all four head characters to 'H', keeping 3 channels
instead of 6 while the one-hot direction restores that information cheaply.
"""
view = HeadlessView()
view.on_game_start(game)
view.render(game)
normalized = [
["H" if c in HEAD_CHARS else c for c in row]
for row in view.board_characters
]
board_vec = encode_board(normalized, CHARACTER_SET_NORMALIZED).transpose(2, 0, 1).flatten()
extras = encode_state(game.state, ["apple_dx", "apple_dy"])
head = game.get_agent_by_name("Snake head")
direction_onehot = np.array(
[1.0 if head.direction == d else 0.0 for d in DIRECTIONS],
dtype=np.float32,
)
return np.concatenate([board_vec, extras, direction_onehot])
def normalized_observation(game):
"""Full board with all head chars collapsed to 'H', plus apple_dx and apple_dy.
Returns board in channel-first (C, H, W) order for the CNN, followed by extras.
Character set: ['@' apple, '*' body, 'H' head] — 3 channels instead of 6.
"""
view = HeadlessView()
view.on_game_start(game)
view.render(game)
normalized = [
["H" if c in HEAD_CHARS else c for c in row]
for row in view.board_characters
]
board_vec = encode_board(normalized, CHARACTER_SET_NORMALIZED).transpose(2, 0, 1).flatten()
extras = encode_state(game.state, ["apple_dx", "apple_dy"])
return np.concatenate([board_vec, extras])
def egocentric_cnn_observation(game):
"""17×17 egocentric window in channel-first (CHW) format for CNN, plus apple_dx/dy."""
view = HeadlessView()
view.on_game_start(game)
view.render(game)
head = game.get_agent_by_name("Snake head")
cropped = egocentric_board(view.board_characters, head.position, RADIUS_WIDE)
board_vec = encode_board(cropped, CHARACTER_SET).transpose(2, 0, 1).flatten()
extras = encode_state(game.state, ["apple_dx", "apple_dy"])
return np.concatenate([board_vec, extras])
def narrow_egocentric_observation(game):
"""7×7 egocentric window (flat, for MLP), plus apple_dx and apple_dy."""
view = HeadlessView()
view.on_game_start(game)
view.render(game)
head = game.get_agent_by_name("Snake head")
cropped = egocentric_board(view.board_characters, head.position, RADIUS_NARROW)
board_vec = encode_board(cropped, CHARACTER_SET).flatten()
extras = encode_state(game.state, ["apple_dx", "apple_dy"])
return np.concatenate([board_vec, extras])
def egocentric_observation(game): def egocentric_observation(game):
"""17×17 window centered on the snake's head, plus apple_dx and apple_dy.""" """17×17 window centered on the snake's head, plus apple_dx and apple_dy."""

1
games/snake/pyproject.toml
View File

@@ -8,3 +8,4 @@ dependencies = ["retro-games>=2.5.0"]
[tool.retro-gamer] [tool.retro-gamer]
actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"] actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"]
reward = "reward" reward = "reward"
character_set = ["@", "*", "<", ">", "^", "v"]

1
games/snake_v1/pyproject.toml
View File

@@ -8,3 +8,4 @@ dependencies = ["retro-games>=2.5.0"]
[tool.retro-gamer] [tool.retro-gamer]
actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"] actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"]
reward = "reward" reward = "reward"
character_set = ["@", "*", "<", ">", "^", "v"]

40
questions.md
View File

@@ -29,37 +29,41 @@
## Checkpoint 4 ## Checkpoint 4
9. **Full board (runs/snake-v1, ep_5000):** Describe the agent's behavior. Does it seem to know 9. In Attempt 1, the agent sees the full 32×16 board as 3,072 numbers—the apple's location
where the apple is? Does it move randomly or with some purpose? is already in there somewhere. In Attempt 2, we supplemented the board with just two extra
numbers: the direction to the apple. Performance tripled. Why did two extra numbers make
such a large difference when the board already contained the apple's location?
10. **Features only (runs/snake-v2, ep_3000):** How does this agent differ from the v1 agent? 10. Attempt 3 added the full board back and switched to a CNN—a more powerful
What is it doing better? What is it doing that leads to shorter episodes? architecture—yet performance was worse than Attempt 1. Why didn't more information
and a more powerful model help?
11. **Final run, early (runs/snake, ep_1300):** This agent uses the egocentric view plus 11. The only difference between Attempt 3 and Attempt 4 is that Attempt 4 shows the agent a
apple_dx/apple_dy. What has it learned that neither v1 nor v2 showed? 17×17 window centered on its own head, rather than the full board. Why did this single
change make such a large difference?
12. **Final run, mature (runs/snake, ep_20000):** What does this agent do well? Where does it 12. When the snake's body gets very long, it becomes important to plan your route so you don't
still make mistakes? get trapped inside your own body. None of our training attempts was very successful at
learning this behavior. Which of the approaches do you think would be most promising for
learning it? Why?
13. In the features-only run (v2), reward rose as episodes got shorter. Why does a snake agent 13. The reward function gives the snake +1 for each step it moves toward the apple and −1 for
that is getting better at finding apples end up with shorter episodes? each step away. Can you think of a way this reward signal might accidentally encourage bad
behavior—especially as the snake grows longer?
14. The egocentric view crops the observation to a 17×17 window centered on the snake's head.
What did the agent gain from this change, and what information did it lose access to?
## Checkpoint 5 ## Checkpoint 5
Answer these questions after completing both training experiments in "Training Frogger." Answer these questions after completing both training experiments in "Training Frogger."
15. **Hypothesis (Attempt 1):** Before training, predict what will happen. Will the agent learn to 14. **Hypothesis (Attempt 1):** Before training, predict what will happen. Will the agent learn to
reach the top of the board? What challenge do you think it will face? reach the top of the board? What challenge do you think it will face?
16. **Evidence (Attempt 1):** Copy the first three and last three lines of `runs/frogger/training.log`. 15. **Evidence (Attempt 1):** Copy the first three and last three lines of `runs/frogger/training.log`.
Did training go as expected? Did training go as expected?
17. **Analysis (Attempt 1):** What did the agent learn to do? Where did it struggle? 16. **Analysis (Attempt 1):** What did the agent learn to do? Where did it struggle?
18. **Experiment (Attempt 2):** What one thing did you change? Write your prediction, show the 17. **Experiment (Attempt 2):** What one thing did you change? Write your prediction, show the
evidence (first and last few log lines), and describe what happened. evidence (first and last few log lines), and describe what happened.
19. Which attempt produced the best agent? What would you try next if you had more time? 18. Which attempt produced the best agent? What would you try next if you had more time?