diff --git a/babysnake/__main__.py b/babysnake/__main__.py deleted file mode 100644 index 47ceec9..0000000 --- a/babysnake/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from babysnake import create_game - -create_game().play() diff --git a/babysnake/pyproject.toml b/babysnake/pyproject.toml deleted file mode 100644 index 6b68000..0000000 --- a/babysnake/pyproject.toml +++ /dev/null @@ -1,4 +0,0 @@ -[tool.retro-gamer] -actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"] -reward = "reward" -observation_function = "babysnake:get_state" diff --git a/forager/__init__.py b/forager/__init__.py deleted file mode 100644 index d445077..0000000 --- a/forager/__init__.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Forager: an 8×8 grid game where an agent collects food. - -The agent moves in four directions collecting food that respawns on collection. -The agent starts with 100 energy. Each step costs 1 energy. Collecting food -restores 40 energy. The episode ends when energy reaches 0. - -Observation features for retro-gamer: - food_dx: (food_x - agent_x) / board_width (positive = food is to the right) - food_dy: (food_y - agent_y) / board_height (positive = food is below) -""" - -from random import randint -from retro.game import Game - -BOARD_SIZE = 8 -START_ENERGY = 100 -FOOD_ENERGY = 40 - - -class Forager: - """The player agent.""" - name = "Forager" - character = '@' - color = "green_on_black" - position = (0, 0) - - UP = (0, -1) - DOWN = (0, 1) - LEFT = (-1, 0) - RIGHT = (1, 0) - - def __init__(self): - self._direction = self.RIGHT - - def handle_keystroke(self, keystroke, game): - if keystroke.name == "KEY_RIGHT": - self._direction = self.RIGHT - elif keystroke.name == "KEY_UP": - self._direction = self.UP - elif keystroke.name == "KEY_LEFT": - self._direction = self.LEFT - elif keystroke.name == "KEY_DOWN": - self._direction = self.DOWN - - def play_turn(self, game): - x, y = self.position - dx, dy = self._direction - new_pos = (x + dx, y + dy) - if game.on_board(new_pos): - self.position = new_pos - - game.state['energy'] -= 1 - game.state['reward'] -= 0.01 - - food = game.get_agent_by_name("Food") - if self.position == food.position: - food.relocate(game) - game.state['energy'] += FOOD_ENERGY - game.state['score'] += 1 - game.state['reward'] += 1.0 - - bw, bh = game.board_size - ax, ay = self.position - fx, fy = game.get_agent_by_name("Food").position - game.state['food_dx'] = (fx - ax) / bw - game.state['food_dy'] = (fy - ay) / bh - - if game.state['energy'] <= 0: - game.end() - - -class Food: - """The food item. Respawns when collected.""" - name = "Food" - character = '*' - color = "yellow_on_black" - position = (0, 0) - - def relocate(self, game): - bw, bh = game.board_size - forager = game.get_agent_by_name("Forager") - while True: - pos = (randint(0, bw - 1), randint(0, bh - 1)) - if pos != forager.position: - self.position = pos - return - - -def create_game(): - """Return a fresh Forager game.""" - forager = Forager() - food = Food() - bw = bh = BOARD_SIZE - game = Game( - [forager, food], - {'score': 0, 'reward': 0.0, 'energy': START_ENERGY, 'food_dx': 0.0, 'food_dy': 0.0}, - board_size=(bw, bh), - framerate=12, - ) - forager.position = (randint(0, bw - 1), randint(0, bh - 1)) - food.relocate(game) - bw, bh = game.board_size - ax, ay = forager.position - fx, fy = food.position - game.state['food_dx'] = (fx - ax) / bw - game.state['food_dy'] = (fy - ay) / bh - return game - - -if __name__ == '__main__': - create_game().play() diff --git a/forager/__main__.py b/forager/__main__.py deleted file mode 100644 index da8b141..0000000 --- a/forager/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from forager import create_game - -create_game().play() diff --git a/forager/pyproject.toml b/forager/pyproject.toml deleted file mode 100644 index a073f61..0000000 --- a/forager/pyproject.toml +++ /dev/null @@ -1,4 +0,0 @@ -[tool.retro-gamer] -actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"] -reward = "reward" -character_set = ["@", "*"] diff --git a/games/__init__.py b/games/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/babysnake/__init__.py b/games/babysnake/__init__.py similarity index 100% rename from babysnake/__init__.py rename to games/babysnake/__init__.py diff --git a/games/babysnake/__main__.py b/games/babysnake/__main__.py new file mode 100644 index 0000000..e7a098d --- /dev/null +++ b/games/babysnake/__main__.py @@ -0,0 +1,3 @@ +from games.babysnake import create_game + +create_game().play() diff --git a/games/babysnake/pyproject.toml b/games/babysnake/pyproject.toml new file mode 100644 index 0000000..3f41acb --- /dev/null +++ b/games/babysnake/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "babysnake" +version = "0.1.0" +description = "BabySnake: a tiny grid game for teaching Q-learning" +requires-python = ">=3.11" +dependencies = ["retro-games>=2.5.0"] + +[tool.retro-gamer] +actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"] +reward = "reward" +observation_function = "games.babysnake:get_state" diff --git a/train_babysnake.py b/games/babysnake/train.py similarity index 95% rename from train_babysnake.py rename to games/babysnake/train.py index ed50783..888a175 100644 --- a/train_babysnake.py +++ b/games/babysnake/train.py @@ -10,13 +10,13 @@ babysnake/pyproject.toml's observation_function) as the environment. """ from q_learning import QLearning -import babysnake +from games import babysnake from retro.input import ProgrammaticInput from retro_gamer import GameEnvironment, GameMetadata def train(): trainer = QLearning() - env = GameEnvironment(babysnake.create_game, GameMetadata.from_pyproject("babysnake")) + env = GameEnvironment(babysnake.create_game, GameMetadata.from_pyproject("games.babysnake")) return trainer.train(env, babysnake.ACTIONS) def watch(Q): diff --git a/games/frogger/__init__.py b/games/frogger/__init__.py new file mode 100644 index 0000000..69fad5c --- /dev/null +++ b/games/frogger/__init__.py @@ -0,0 +1,161 @@ +"""Frogger: guide a frog across busy traffic lanes to reach the far side. + +The frog starts at the bottom row. Cars move across the lanes between the +start and goal. Each lane has cars moving at different speeds in alternating +directions. The frog earns +10 for each row advanced, +50 for reaching the +top row, and -10 for being hit by a car or falling off the edge. Episodes end +when the frog reaches the top, gets hit, or energy runs out. +""" + +from random import randint +from retro.game import Game + +BOARD_WIDTH = 20 +BOARD_HEIGHT = 12 +NUM_LANES = BOARD_HEIGHT - 2 +START_ENERGY = 200 + + +class Frog: + name = "Frog" + character = 'O' + color = "green_on_black" + position = (0, 0) + + UP = (0, -1) + DOWN = (0, 1) + LEFT = (-1, 0) + RIGHT = (1, 0) + + def __init__(self): + self._direction = self.UP + + def handle_keystroke(self, keystroke, game): + if keystroke.name == "KEY_UP": + self._direction = self.UP + elif keystroke.name == "KEY_DOWN": + self._direction = self.DOWN + elif keystroke.name == "KEY_LEFT": + self._direction = self.LEFT + elif keystroke.name == "KEY_RIGHT": + self._direction = self.RIGHT + + def play_turn(self, game): + bw, bh = game.board_size + x, y = self.position + dx, dy = self._direction + nx, ny = x + dx, y + dy + + if not (0 <= nx < bw): + game.state['reward'] -= 10 + game.state['energy'] -= 50 + self._reset(game) + return + + if not (0 <= ny < bh): + if ny < 0: + game.state['score'] += 50 + game.state['reward'] += 50 + else: + game.state['reward'] -= 5 + self._reset(game) + return + + prev_y = y + self.position = (nx, ny) + game.state['energy'] -= 1 + game.state['reward'] -= 0.01 + + if ny < prev_y: + advancement = prev_y - ny + game.state['score'] += advancement * 10 + game.state['reward'] += advancement * 5 + + for agent in game.agents: + if hasattr(agent, '_is_car') and agent.position == self.position: + game.state['reward'] -= 10 + game.state['energy'] -= 50 + self._reset(game) + return + + bw, bh = game.board_size + fx, fy = self.position + game.state['frog_x'] = fx / bw + game.state['frog_y'] = fy / bh + + if game.state['energy'] <= 0: + game.end() + + def _reset(self, game): + bw, bh = game.board_size + self.position = (bw // 2, bh - 1) + self._direction = self.UP + if game.state['energy'] <= 0: + game.end() + + +class Car: + _is_car = True + character = 'X' + color = "red_on_black" + + def __init__(self, lane, speed, direction, start_x, board_width): + self.name = f"Car {lane}_{start_x}" + self._lane = lane + self._speed = speed + self._direction = direction + self._board_width = board_width + self._step = 0 + self.position = (start_x, lane) + + def play_turn(self, game): + self._step += 1 + if self._step < self._speed: + return + self._step = 0 + x, y = self.position + x = (x + self._direction) % self._board_width + self.position = (x, y) + + frog = game.get_agent_by_name("Frog") + if frog.position == self.position: + frog._reset(game) + game.state['reward'] -= 10 + game.state['energy'] -= 50 + + +def create_game(): + bw, bh = BOARD_WIDTH, BOARD_HEIGHT + frog = Frog() + frog.position = (bw // 2, bh - 1) + + agents = [frog] + car_id = 0 + for lane_idx, row in enumerate(range(1, bh - 1)): + direction = 1 if lane_idx % 2 == 0 else -1 + speed = 2 + (lane_idx % 3) + num_cars = 2 + (lane_idx % 3) + spacing = bw // num_cars + for i in range(num_cars): + start_x = (i * spacing + lane_idx * 3) % bw + agents.append(Car(row, speed, direction, start_x, bw)) + car_id += 1 + + game = Game( + agents, + { + 'score': 0, + 'reward': 0.0, + 'energy': START_ENERGY, + 'frog_x': (bw // 2) / bw, + 'frog_y': (bh - 1) / bh, + }, + board_size=(bw, bh), + framerate=8, + show_state=['score', 'energy'], + ) + return game + + +if __name__ == '__main__': + create_game().play() diff --git a/games/frogger/__main__.py b/games/frogger/__main__.py new file mode 100644 index 0000000..8ec44db --- /dev/null +++ b/games/frogger/__main__.py @@ -0,0 +1,3 @@ +from games.frogger import create_game + +create_game().play() diff --git a/games/frogger/pyproject.toml b/games/frogger/pyproject.toml new file mode 100644 index 0000000..be794a1 --- /dev/null +++ b/games/frogger/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "frogger" +version = "0.1.0" +description = "Frogger: guide a frog across traffic lanes to train an RL agent" +requires-python = ">=3.11" +dependencies = ["retro-games>=2.5.0"] + +[tool.retro-gamer] +actions = ["KEY_UP", "KEY_DOWN", "KEY_LEFT", "KEY_RIGHT"] +reward = "reward" +character_set = ["O", "X"] diff --git a/games/snake/__init__.py b/games/snake/__init__.py new file mode 100644 index 0000000..f694fe7 --- /dev/null +++ b/games/snake/__init__.py @@ -0,0 +1,36 @@ +from random import randint, choice +from .snake import SnakeHead +from .apple import Apple +from retro.game import Game + +def create_game(): + """Return a fresh, initialized Snake game.""" + head = SnakeHead() + apple = Apple() + game = Game( + [head, apple], + {'score': 0, 'reward': 0, 'energy': 150, 'apple_dx': 0.0, 'apple_dy': 0.0}, + board_size=(32, 16), + framerate=12, + ) + bw, bh = game.board_size + head.position = (randint(1, bw - 2), randint(1, bh - 2)) + direction, character = choice([ + (SnakeHead.RIGHT, '>'), + (SnakeHead.UP, '^'), + (SnakeHead.LEFT, '<'), + (SnakeHead.DOWN, 'v'), + ]) + head.direction = direction + head.character = character + apple.relocate(game) + hx, hy = head.position + ax, ay = apple.position + game.state['apple_dx'] = (ax - hx) / bw + game.state['apple_dy'] = (ay - hy) / bh + return game + + +if __name__ == '__main__': + create_game().play() + diff --git a/games/snake/__main__.py b/games/snake/__main__.py new file mode 100644 index 0000000..af516a0 --- /dev/null +++ b/games/snake/__main__.py @@ -0,0 +1,3 @@ +from games.snake import create_game + +create_game().play() diff --git a/games/snake/apple.py b/games/snake/apple.py new file mode 100644 index 0000000..89610fd --- /dev/null +++ b/games/snake/apple.py @@ -0,0 +1,49 @@ +from random import randint + +class Apple: + """An agent representing the Apple. + Note how Apple doesn't have ``play_turn`` or + ``handle_keystroke`` methods: the Apple doesn't need to do + anything in this game. It just sits there waiting to get + eaten. + + Attributes: + name: "Apple" + character: '@' + color: "red_on_black" (`Here's documentation on how colors + work `_ + position: (0, 0). The Apple will choose a random position + as soon as the game starts, but it needs an initial + position to be assigned. + + """ + name = "Apple" + character = '@' + color = "red_on_black" + position = (0, 0) + + def relocate(self, game): + """Sets position to a random empty position. This method is + called whenever the snake's head touches the apple. + + Arguments: + game (Game): The current game. + """ + self.position = self.random_empty_position(game) + + def random_empty_position(self, game): + """Returns a randomly-selected empty position. Uses a very + simple algorithm: Get the game's board size, choose a + random x-value between 0 and the board width, and choose + a random y-value between 0 and the board height. Now use + the game to check whether any Agents are occupying this + position. If so, keep randomly choosing a new position + until the position is empty. + """ + bw, bh = game.board_size + occupied_positions = game.get_agents_by_position() + while True: + position = (randint(0, bw-1), randint(0, bh-1)) + if position not in occupied_positions: + return position + diff --git a/games/snake/observation.py b/games/snake/observation.py new file mode 100644 index 0000000..a13468c --- /dev/null +++ b/games/snake/observation.py @@ -0,0 +1,18 @@ +import numpy as np +from retro.views.headless import HeadlessView +from retro_gamer.observation import egocentric_board, encode_board, encode_state + +CHARACTER_SET = ["@", "*", ">", "<", "^", "v"] +RADIUS = 8 + + +def egocentric_observation(game): + """17×17 window centered on the snake's head, plus apple_dx and apple_dy.""" + view = HeadlessView() + view.on_game_start(game) + view.render(game) + head = game.get_agent_by_name("Snake head") + cropped = egocentric_board(view.board_characters, head.position, RADIUS) + board_vec = encode_board(cropped, CHARACTER_SET).flatten() + extras = encode_state(game.state, ["apple_dx", "apple_dy"]) + return np.concatenate([board_vec, extras]) diff --git a/games/snake/pyproject.toml b/games/snake/pyproject.toml new file mode 100644 index 0000000..b090c2e --- /dev/null +++ b/games/snake/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "snake" +version = "0.1.0" +description = "Snake: a classic game for training RL agents" +requires-python = ">=3.11" +dependencies = ["retro-games>=2.5.0"] + +[tool.retro-gamer] +actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"] +reward = "reward" diff --git a/games/snake/snake.py b/games/snake/snake.py new file mode 100644 index 0000000..0870d4f --- /dev/null +++ b/games/snake/snake.py @@ -0,0 +1,160 @@ + +class SnakeHead: + """An Agent representing the snake's head. When the game starts, you control + the snake head using the arrow keys. The SnakeHead always has a direction, and + will keep moving in that direction every turn. When you press an arrow key, + you change the SnakeHead's direction. + + Attributes: + name: "Snake head" + position: (0,0) + character: ``'v'`` Depending on the snake head's direction, its character + changes to ``'<'``, ``'^'``, ``'>'``, or ``'v'``. + next_segment: Initially ``None``, this is a reference to a SnakeBodySegment. + growing: When set to True, the snake will grow a new segment on its next move. + """ + RIGHT = (1, 0) + UP = (0, -1) + LEFT = (-1, 0) + DOWN = (0, 1) + name = "Snake head" + position = (0, 0) + direction = DOWN + character = 'v' + next_segment = None + growing = False + + def play_turn(self, game): + """On each turn, the snake head uses its position and direction to figure out + its next position. If the snake head is able to move there (it's on the board and + not occuppied by part of the snake's body), it moves. + + Then, if the snake head is on the Apple, the Apple moves to a new random position + and ``growing`` is set to True. + + Now we need to deal with two situations. First, if ``next_segment`` is not None, there is + a SnakeBodySegment attached to the head. We need the body to follow the head, + so we call ``self.next_segment.move``, passing the head's old position + (this will be the body's new position), a reference to the game, and a value for + ``growing``. If the snake needs to grow, we need to pass this information along + the body until it reaches the tail--this is where the next segment will be attached. + + If there is no ``next_segment`` but ``self.growing`` is True, it's time to add + a body! We set ``self.next_segment`` to a new SnakeBodySegment, set its + position to the head's old position, and add it to the game. We also add 1 to the + game's score. + """ + x, y = self.position + dx, dy = self.direction + next_pos = (x + dx, y + dy) + if self.can_move(next_pos, game): + apple = game.get_agent_by_name("Apple") + ax, ay = apple.position + old_dist = abs(x - ax) + abs(y - ay) + new_dist = abs(next_pos[0] - ax) + abs(next_pos[1] - ay) + game.state['reward'] += old_dist - new_dist # +1 toward apple, -1 away + game.state['energy'] -= 1 + self.position = next_pos + if self.is_on_apple(self.position, game): + apple.relocate(game) + self.growing = True + game.state['score'] += 50 + game.state['reward'] += 50 + game.state['energy'] = 150 + if self.next_segment: + self.next_segment.move((x, y), game, growing=self.growing) + elif self.growing: + self.next_segment = SnakeBodySegment(1, (x, y)) + game.add_agent(self.next_segment) + self.growing = False + bw, bh = game.board_size + hx, hy = self.position + ax, ay = apple.position + game.state['apple_dx'] = (ax - hx) / bw + game.state['apple_dy'] = (ay - hy) / bh + if game.state['energy'] <= 0: + game.state['reward'] -= 10 + game.end() + else: + game.state['reward'] -= 10 + game.end() + + def handle_keystroke(self, keystroke, game): + """Checks whether one of the arrow keys has been pressed. + If so, sets the SnakeHead's direction and character. + """ + x, y = self.position + if keystroke.name == "KEY_RIGHT": + self.direction = self.RIGHT + self.character = '>' + elif keystroke.name == "KEY_UP": + self.direction = self.UP + self.character = '^' + elif keystroke.name == "KEY_LEFT": + self.direction = self.LEFT + self.character = '<' + elif keystroke.name == "KEY_DOWN": + self.direction = self.DOWN + self.character = 'v' + + def can_move(self, position, game): + on_board = game.on_board(position) + empty = game.is_empty(position) + on_apple = self.is_on_apple(position, game) + return on_board and (empty or on_apple) + + def is_on_apple(self, position, game): + apple = game.get_agent_by_name("Apple") + return apple.position == position + +class SnakeBodySegment: + """Finally, we need an Agent for the snake's body segments. + SnakeBodySegment doesn't have ``play_turn`` or ``handle_keystroke`` methods because + it never does anything on its own. It only moves when the SnakeHead, or the previous + segment, tells it to move. + + Arguments: + segment_id (int): Keeps track of how far back this segment is from the head. + This is used to give the segment a unique name, and also to keep track + of how many points the player earns for eating the next apple. + position (int, int): The initial position. + + Attributes: + character: '*' + next_segment: Initially ``None``, this is a reference to a SnakeBodySegment + when this segment is not the last one in the snake's body. + + """ + character = '*' + next_segment = None + + def __init__(self, segment_id, position): + self.segment_id = segment_id + self.name = f"Snake body segment {segment_id}" + self.position = position + + def move(self, new_position, game, growing=False): + """When SnakeHead moves, it sets off a chain reaction, moving all its + body segments. Whenever the head or a body segment has another segment + (``next_segment``), it calls that segment's ``move`` method. + + This method updates the SnakeBodySegment's position. Then, if + ``self.next_segment`` is not None, calls that segment's ``move`` method. + If there is no next segment and ``growing`` is True, then we set + ``self.next_segment`` to a new SnakeBodySegment in this segment's old + position, and update the game's score. + + Arguments: + new_position (int, int): The new position. + game (Game): A reference to the current game. + growing (bool): (Default False) When True, the snake needs to + add a new segment. + """ + old_position = self.position + self.position = new_position + if self.next_segment: + self.next_segment.move(old_position, game, growing=growing) + elif growing: + self.next_segment = SnakeBodySegment(self.segment_id + 1, old_position) + game.add_agent(self.next_segment) + diff --git a/games/snake_v1/__init__.py b/games/snake_v1/__init__.py new file mode 100644 index 0000000..1368b84 --- /dev/null +++ b/games/snake_v1/__init__.py @@ -0,0 +1,92 @@ +from random import randint, choice +from games.snake.apple import Apple +from games.snake.snake import SnakeBodySegment +from retro.game import Game + + +class SnakeHead: + RIGHT = (1, 0) + UP = (0, -1) + LEFT = (-1, 0) + DOWN = (0, 1) + name = "Snake head" + position = (0, 0) + direction = DOWN + character = 'v' + next_segment = None + growing = False + + def play_turn(self, game): + x, y = self.position + dx, dy = self.direction + next_pos = (x + dx, y + dy) + if self.can_move(next_pos, game): + apple = game.get_agent_by_name("Apple") + ax, ay = apple.position + old_dist = abs(x - ax) + abs(y - ay) + new_dist = abs(next_pos[0] - ax) + abs(next_pos[1] - ay) + game.state['reward'] += old_dist - new_dist + game.state['energy'] -= 1 + self.position = next_pos + if self.is_on_apple(self.position, game): + apple.relocate(game) + self.growing = True + game.state['score'] += 50 + game.state['reward'] += 50 + game.state['energy'] = 150 + if self.next_segment: + self.next_segment.move((x, y), game, growing=self.growing) + elif self.growing: + self.next_segment = SnakeBodySegment(1, (x, y)) + game.add_agent(self.next_segment) + self.growing = False + if game.state['energy'] <= 0: + game.state['reward'] -= 10 + game.end() + else: + game.state['reward'] -= 10 + game.end() + + def handle_keystroke(self, keystroke, game): + if keystroke.name == "KEY_RIGHT": + self.direction = self.RIGHT + self.character = '>' + elif keystroke.name == "KEY_UP": + self.direction = self.UP + self.character = '^' + elif keystroke.name == "KEY_LEFT": + self.direction = self.LEFT + self.character = '<' + elif keystroke.name == "KEY_DOWN": + self.direction = self.DOWN + self.character = 'v' + + def can_move(self, position, game): + return game.on_board(position) and (game.is_empty(position) or self.is_on_apple(position, game)) + + def is_on_apple(self, position, game): + return game.get_agent_by_name("Apple").position == position + + +def create_game(): + head = SnakeHead() + apple = Apple() + game = Game( + [head, apple], + {'score': 0, 'reward': 0, 'energy': 150}, + board_size=(32, 16), + framerate=12, + show_state=['score'], + ) + bw, bh = game.board_size + head.position = (randint(1, bw - 2), randint(1, bh - 2)) + direction, character = choice([ + (SnakeHead.RIGHT, '>'), + (SnakeHead.UP, '^'), + (SnakeHead.LEFT, '<'), + (SnakeHead.DOWN, 'v'), + ]) + head.direction = direction + head.character = character + apple.relocate(game) + return game diff --git a/games/snake_v1/pyproject.toml b/games/snake_v1/pyproject.toml new file mode 100644 index 0000000..d6075b1 --- /dev/null +++ b/games/snake_v1/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "snake_v1" +version = "0.1.0" +description = "Snake (v1): full board, no extra features" +requires-python = ">=3.11" +dependencies = ["retro-games>=2.5.0"] + +[tool.retro-gamer] +actions = ["KEY_RIGHT", "KEY_UP", "KEY_LEFT", "KEY_DOWN"] +reward = "reward" diff --git a/pyproject.toml b/pyproject.toml index 1f92655..98e358e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ version = "0.1.0" description = "Reinforcement learning lab" requires-python = ">=3.11" dependencies = [ - "retro-games>=2.4.0", - "retro-gamer>=0.2.0", + "retro-games>=2.5.0", + "retro-gamer>=0.3.0", ] [build-system] @@ -13,4 +13,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["babysnake", "forager"] +packages = ["games", "q_learning"] + +[tool.uv.sources] +retro-gamer = { path = "../../packages/retro-gamer", editable = true } diff --git a/questions.md b/questions.md index 4ef7856..525836c 100644 --- a/questions.md +++ b/questions.md @@ -26,3 +26,40 @@ read the policy? For a given state, does the highest Q-value point toward the food? 8. How does the trained agent's behavior compare to the reasoning you wrote down in question 1? + +## Checkpoint 4 + +9. **Full board (runs/snake-v1, ep_5000):** Describe the agent's behavior. Does it seem to know + where the apple is? Does it move randomly or with some purpose? + +10. **Features only (runs/snake-v2, ep_3000):** How does this agent differ from the v1 agent? + What is it doing better? What is it doing that leads to shorter episodes? + +11. **Final run, early (runs/snake, ep_1300):** This agent uses the egocentric view plus + apple_dx/apple_dy. What has it learned that neither v1 nor v2 showed? + +12. **Final run, mature (runs/snake, ep_20000):** What does this agent do well? Where does it + still make mistakes? + +13. In the features-only run (v2), reward rose as episodes got shorter. Why does a snake agent + that is getting better at finding apples end up with shorter episodes? + +14. The egocentric view crops the observation to a 17×17 window centered on the snake's head. + What did the agent gain from this change, and what information did it lose access to? + +## Checkpoint 5 + +Answer these questions after completing both training experiments in "Training Frogger." + +15. **Hypothesis (Attempt 1):** Before training, predict what will happen. Will the agent learn to + reach the top of the board? What challenge do you think it will face? + +16. **Evidence (Attempt 1):** Copy the first three and last three lines of `runs/frogger/training.log`. + Did training go as expected? + +17. **Analysis (Attempt 1):** What did the agent learn to do? Where did it struggle? + +18. **Experiment (Attempt 2):** What one thing did you change? Write your prediction, show the + evidence (first and last few log lines), and describe what happened. + +19. Which attempt produced the best agent? What would you try next if you had more time? diff --git a/runs/snake/checkpoints/ep_1300.pt b/runs/snake/checkpoints/ep_1300.pt deleted file mode 100644 index 88192e3..0000000 Binary files a/runs/snake/checkpoints/ep_1300.pt and /dev/null differ diff --git a/runs/snake/checkpoints/ep_20000.pt b/runs/snake/checkpoints/ep_20000.pt deleted file mode 100644 index e32ecc3..0000000 Binary files a/runs/snake/checkpoints/ep_20000.pt and /dev/null differ diff --git a/runs/snake/checkpoints/ep_4000.pt b/runs/snake/checkpoints/ep_4000.pt deleted file mode 100644 index 9e92e38..0000000 Binary files a/runs/snake/checkpoints/ep_4000.pt and /dev/null differ diff --git a/runs/snake/config.toml b/runs/snake/config.toml deleted file mode 100644 index 3bebc68..0000000 --- a/runs/snake/config.toml +++ /dev/null @@ -1,53 +0,0 @@ -[game] -module = "retro.examples.snake" - -[metadata] -actions = [ - "KEY_RIGHT", - "KEY_UP", - "KEY_LEFT", - "KEY_DOWN", -] -reward = "reward" -extras_size = 2 -board_size = [ - 17, - 17, -] -character_set = [ - "@", - "*", - ">", - "<", - "^", - "v", -] -observation_function = "snake_observation:egocentric_observation" - -[preprocessing] -spatial = false -board = true -observe_state = [] - -[model] -hidden_sizes = [ - 128, - 64, -] - -[training] -learning_rate = 0.0001 -learning_rate_decay = 0.9999 -gamma = 0.99 -epsilon = 1.0 -epsilon_decay = 0.9997 -epsilon_min = 0.05 -batch_size = 64 -memory_capacity = 50000 -target_update_freq = 500 -train_every = 4 -training_episodes = 20000 -prioritize_experiences = true -exploration_turns = 200 -unknown_character_strategy = "ignore" -max_turns_per_episode = 2000 diff --git a/runs/snake/training.log b/runs/snake/training.log deleted file mode 100644 index f8878b0..0000000 --- a/runs/snake/training.log +++ /dev/null @@ -1,213 +0,0 @@ -[INIT] === Network Architecture === -[INIT] Board: 17×17, character set: 6 chars (one-hot per cell) -[INIT] Observed state features: 2 | Actions (incl. no-op): 5 -[INIT] spatial=False → using MLP architecture -[INIT] Rationale: the board encodes UI/status rather than a spatial scene; -[INIT] a flat MLP over the full observation is sufficient. -[INIT] MLP: 1736 → 128 → 64 → 5 -[INIT] Hidden layers: 2 | Layer sizes: [128, 64] -[INIT] Output: 5 Q-values -[INIT] Actions: ['KEY_RIGHT', 'KEY_UP', 'KEY_LEFT', 'KEY_DOWN'] + (no-op) -[INIT] Device: mps - -=== Training started | 2026-06-23 22:39:07 === -[ep_0100] ep=0001-0100 avg_reward=-7.8 avg_steps=49 epsilon=0.970 avg_loss=0.8 time=0m06s total=0m06s -[ep_0200] ep=0101-0200 avg_reward=-5.7 avg_steps=51 epsilon=0.942 avg_loss=0.9 time=0m07s total=0m13s -[ep_0300] ep=0201-0300 avg_reward=+0.1 avg_steps=49 epsilon=0.914 avg_loss=1.2 time=0m06s total=0m20s -[ep_0400] ep=0301-0400 avg_reward=+7.9 avg_steps=70 epsilon=0.887 avg_loss=1.6 time=0m09s total=0m30s -[ep_0500] ep=0401-0500 avg_reward=+12.8 avg_steps=57 epsilon=0.861 avg_loss=1.9 time=0m08s total=0m38s -[ep_0600] ep=0501-0600 avg_reward=+20.5 avg_steps=57 epsilon=0.835 avg_loss=2.3 time=0m08s total=0m47s -[ep_0700] ep=0601-0700 avg_reward=+23.7 avg_steps=62 epsilon=0.811 avg_loss=2.8 time=0m10s total=0m58s -[ep_0800] ep=0701-0800 avg_reward=+21.2 avg_steps=56 epsilon=0.787 avg_loss=3.4 time=0m10s total=1m08s -[ep_0900] ep=0801-0900 avg_reward=+30.1 avg_steps=55 epsilon=0.763 avg_loss=4.1 time=0m10s total=1m19s -[ep_1000] ep=0901-1000 avg_reward=+33.6 avg_steps=59 epsilon=0.741 avg_loss=4.6 time=0m11s total=1m31s -[ep_1100] ep=1001-1100 avg_reward=+36.0 avg_steps=58 epsilon=0.719 avg_loss=5.4 time=0m11s total=1m43s -[ep_1200] ep=1101-1200 avg_reward=+34.0 avg_steps=50 epsilon=0.698 avg_loss=6.1 time=0m10s total=1m53s -[ep_1300] ep=1201-1300 avg_reward=+31.6 avg_steps=49 epsilon=0.677 avg_loss=6.7 time=0m09s total=2m03s -[ep_1400] ep=1301-1400 avg_reward=+30.4 avg_steps=60 epsilon=0.657 avg_loss=7.2 time=0m12s total=2m15s -[ep_1500] ep=1401-1500 avg_reward=+28.0 avg_steps=65 epsilon=0.638 avg_loss=7.6 time=0m13s total=2m28s -[ep_1600] ep=1501-1600 avg_reward=+16.3 avg_steps=68 epsilon=0.619 avg_loss=7.6 time=0m14s total=2m42s -[ep_1700] ep=1601-1700 avg_reward=+10.1 avg_steps=76 epsilon=0.600 avg_loss=7.7 time=0m18s total=3m01s -[ep_1800] ep=1701-1800 avg_reward=+6.9 avg_steps=89 epsilon=0.583 avg_loss=7.6 time=0m20s total=3m21s -[ep_1900] ep=1801-1900 avg_reward=+1.9 avg_steps=85 epsilon=0.565 avg_loss=7.4 time=0m16s total=3m38s -[ep_2000] ep=1901-2000 avg_reward=+1.1 avg_steps=93 epsilon=0.549 avg_loss=6.9 time=0m18s total=3m57s -[ep_2100] ep=2001-2100 avg_reward=+1.4 avg_steps=105 epsilon=0.533 avg_loss=6.2 time=0m21s total=4m18s -[ep_2200] ep=2101-2200 avg_reward=-0.8 avg_steps=83 epsilon=0.517 avg_loss=5.5 time=0m16s total=4m34s -[ep_2300] ep=2201-2300 avg_reward=-3.8 avg_steps=87 epsilon=0.502 avg_loss=5.1 time=0m17s total=4m52s -[ep_2400] ep=2301-2400 avg_reward=-0.1 avg_steps=89 epsilon=0.487 avg_loss=4.6 time=0m17s total=5m09s -[ep_2500] ep=2401-2500 avg_reward=-2.5 avg_steps=94 epsilon=0.472 avg_loss=4.1 time=0m18s total=5m28s -[ep_2600] ep=2501-2600 avg_reward=+1.2 avg_steps=97 epsilon=0.458 avg_loss=3.7 time=0m19s total=5m48s -[ep_2700] ep=2601-2700 avg_reward=+8.0 avg_steps=85 epsilon=0.445 avg_loss=3.4 time=0m16s total=6m05s -[ep_2800] ep=2701-2800 avg_reward=+5.2 avg_steps=83 epsilon=0.432 avg_loss=2.9 time=0m16s total=6m21s -[ep_2900] ep=2801-2900 avg_reward=+8.4 avg_steps=89 epsilon=0.419 avg_loss=2.7 time=0m17s total=6m39s -[ep_3000] ep=2901-3000 avg_reward=+4.0 avg_steps=98 epsilon=0.407 avg_loss=2.4 time=0m19s total=6m59s -[ep_3100] ep=3001-3100 avg_reward=+6.0 avg_steps=80 epsilon=0.394 avg_loss=2.1 time=0m16s total=7m15s -[ep_3200] ep=3101-3200 avg_reward=+7.0 avg_steps=86 epsilon=0.383 avg_loss=2.0 time=0m17s total=7m33s -[ep_3300] ep=3201-3300 avg_reward=+4.3 avg_steps=93 epsilon=0.372 avg_loss=1.8 time=0m19s total=7m52s -[ep_3400] ep=3301-3400 avg_reward=+2.7 avg_steps=98 epsilon=0.361 avg_loss=1.8 time=0m20s total=8m12s -[ep_3500] ep=3401-3500 avg_reward=+7.7 avg_steps=99 epsilon=0.350 avg_loss=1.6 time=0m20s total=8m32s -[ep_3600] ep=3501-3600 avg_reward=+21.0 avg_steps=86 epsilon=0.340 avg_loss=1.5 time=0m17s total=8m50s -[ep_3700] ep=3601-3700 avg_reward=+30.8 avg_steps=67 epsilon=0.330 avg_loss=1.4 time=0m13s total=9m03s -[ep_3800] ep=3701-3800 avg_reward=+38.4 avg_steps=53 epsilon=0.320 avg_loss=1.3 time=0m10s total=9m14s -[ep_3900] ep=3801-3900 avg_reward=+48.0 avg_steps=49 epsilon=0.310 avg_loss=1.4 time=0m09s total=9m24s -[ep_4000] ep=3901-4000 avg_reward=+53.2 avg_steps=42 epsilon=0.301 avg_loss=1.4 time=0m08s total=9m33s -[ep_4100] ep=4001-4100 avg_reward=+49.1 avg_steps=47 epsilon=0.292 avg_loss=1.5 time=0m09s total=9m42s -[ep_4200] ep=4101-4200 avg_reward=+49.9 avg_steps=50 epsilon=0.284 avg_loss=1.6 time=0m10s total=9m53s -[ep_4300] ep=4201-4300 avg_reward=+54.5 avg_steps=48 epsilon=0.275 avg_loss=1.7 time=0m09s total=10m02s -[ep_4400] ep=4301-4400 avg_reward=+57.5 avg_steps=48 epsilon=0.267 avg_loss=1.8 time=0m09s total=10m12s -[ep_4500] ep=4401-4500 avg_reward=+53.6 avg_steps=47 epsilon=0.259 avg_loss=1.9 time=0m09s total=10m22s -[ep_4600] ep=4501-4600 avg_reward=+59.3 avg_steps=44 epsilon=0.252 avg_loss=1.9 time=0m09s total=10m31s -[ep_4700] ep=4601-4700 avg_reward=+60.2 avg_steps=39 epsilon=0.244 avg_loss=1.8 time=0m08s total=10m39s -[ep_4800] ep=4701-4800 avg_reward=+58.6 avg_steps=40 epsilon=0.237 avg_loss=1.8 time=0m08s total=10m47s -[ep_4900] ep=4801-4900 avg_reward=+64.5 avg_steps=54 epsilon=0.230 avg_loss=1.9 time=0m11s total=10m59s -[ep_5000] ep=4901-5000 avg_reward=+72.9 avg_steps=37 epsilon=0.223 avg_loss=1.9 time=0m07s total=11m06s -[ep_5100] ep=5001-5100 avg_reward=+72.2 avg_steps=36 epsilon=0.216 avg_loss=1.9 time=0m07s total=11m14s -[ep_5200] ep=5101-5200 avg_reward=+71.6 avg_steps=39 epsilon=0.210 avg_loss=1.9 time=0m08s total=11m22s -[ep_5300] ep=5201-5300 avg_reward=+67.9 avg_steps=34 epsilon=0.204 avg_loss=1.9 time=0m07s total=11m29s -[ep_5400] ep=5301-5400 avg_reward=+89.2 avg_steps=43 epsilon=0.198 avg_loss=1.9 time=0m09s total=11m38s -[ep_5500] ep=5401-5500 avg_reward=+92.0 avg_steps=42 epsilon=0.192 avg_loss=2.0 time=0m08s total=11m47s -[ep_5600] ep=5501-5600 avg_reward=+86.4 avg_steps=35 epsilon=0.186 avg_loss=2.1 time=0m07s total=11m55s -[ep_5700] ep=5601-5700 avg_reward=+92.5 avg_steps=37 epsilon=0.181 avg_loss=2.2 time=0m07s total=12m02s -[ep_5800] ep=5701-5800 avg_reward=+98.5 avg_steps=45 epsilon=0.175 avg_loss=2.3 time=0m09s total=12m12s -[ep_5900] ep=5801-5900 avg_reward=+100.9 avg_steps=39 epsilon=0.170 avg_loss=2.5 time=0m08s total=12m20s -[ep_6000] ep=5901-6000 avg_reward=+95.3 avg_steps=40 epsilon=0.165 avg_loss=2.7 time=0m08s total=12m28s -[ep_6100] ep=6001-6100 avg_reward=+92.4 avg_steps=40 epsilon=0.160 avg_loss=2.9 time=0m08s total=12m37s -[ep_6200] ep=6101-6200 avg_reward=+100.2 avg_steps=45 epsilon=0.156 avg_loss=3.1 time=0m09s total=12m46s -[ep_6300] ep=6201-6300 avg_reward=+96.4 avg_steps=43 epsilon=0.151 avg_loss=3.5 time=0m09s total=12m56s -[ep_6400] ep=6301-6400 avg_reward=+107.5 avg_steps=45 epsilon=0.147 avg_loss=3.9 time=0m09s total=13m05s -[ep_6500] ep=6401-6500 avg_reward=+87.4 avg_steps=38 epsilon=0.142 avg_loss=4.1 time=0m08s total=13m13s -[ep_6600] ep=6501-6600 avg_reward=+126.1 avg_steps=51 epsilon=0.138 avg_loss=4.7 time=0m10s total=13m24s -[ep_6700] ep=6601-6700 avg_reward=+118.6 avg_steps=42 epsilon=0.134 avg_loss=5.1 time=0m08s total=13m33s -[ep_6800] ep=6701-6800 avg_reward=+117.7 avg_steps=44 epsilon=0.130 avg_loss=5.7 time=0m09s total=13m42s -[ep_6900] ep=6801-6900 avg_reward=+154.6 avg_steps=56 epsilon=0.126 avg_loss=6.2 time=0m11s total=13m54s -[ep_7000] ep=6901-7000 avg_reward=+129.3 avg_steps=44 epsilon=0.122 avg_loss=6.8 time=0m09s total=14m04s -[ep_7100] ep=7001-7100 avg_reward=+114.2 avg_steps=40 epsilon=0.119 avg_loss=7.2 time=0m08s total=14m12s -[ep_7200] ep=7101-7200 avg_reward=+147.2 avg_steps=52 epsilon=0.115 avg_loss=7.6 time=0m11s total=14m24s -[ep_7300] ep=7201-7300 avg_reward=+122.9 avg_steps=45 epsilon=0.112 avg_loss=8.1 time=0m09s total=14m33s -[ep_7400] ep=7301-7400 avg_reward=+136.2 avg_steps=47 epsilon=0.109 avg_loss=8.4 time=0m10s total=14m43s -[ep_7500] ep=7401-7500 avg_reward=+151.2 avg_steps=51 epsilon=0.105 avg_loss=8.5 time=0m11s total=14m54s -[ep_7600] ep=7501-7600 avg_reward=+150.5 avg_steps=56 epsilon=0.102 avg_loss=8.5 time=0m11s total=15m06s -[ep_7700] ep=7601-7700 avg_reward=+120.3 avg_steps=46 epsilon=0.099 avg_loss=8.6 time=0m09s total=15m16s -[ep_7800] ep=7701-7800 avg_reward=+121.5 avg_steps=47 epsilon=0.096 avg_loss=8.5 time=0m09s total=15m26s -[ep_7900] ep=7801-7900 avg_reward=+160.6 avg_steps=60 epsilon=0.093 avg_loss=8.7 time=0m12s total=15m39s -[ep_8000] ep=7901-8000 avg_reward=+134.2 avg_steps=53 epsilon=0.091 avg_loss=8.6 time=0m11s total=15m50s -[ep_8100] ep=8001-8100 avg_reward=+141.0 avg_steps=51 epsilon=0.088 avg_loss=8.5 time=0m10s total=16m01s -[ep_8200] ep=8101-8200 avg_reward=+167.7 avg_steps=57 epsilon=0.085 avg_loss=8.2 time=0m12s total=16m13s -[ep_8300] ep=8201-8300 avg_reward=+165.9 avg_steps=56 epsilon=0.083 avg_loss=8.0 time=0m11s total=16m25s -[ep_8400] ep=8301-8400 avg_reward=+199.9 avg_steps=71 epsilon=0.080 avg_loss=7.9 time=0m15s total=16m40s -[ep_8500] ep=8401-8500 avg_reward=+196.7 avg_steps=66 epsilon=0.078 avg_loss=7.5 time=0m14s total=16m54s -[ep_8600] ep=8501-8600 avg_reward=+218.4 avg_steps=71 epsilon=0.076 avg_loss=7.4 time=0m15s total=17m10s -[ep_8700] ep=8601-8700 avg_reward=+204.1 avg_steps=74 epsilon=0.074 avg_loss=7.1 time=0m15s total=17m25s -[ep_8800] ep=8701-8800 avg_reward=+216.2 avg_steps=70 epsilon=0.071 avg_loss=6.8 time=0m14s total=17m40s -[ep_8900] ep=8801-8900 avg_reward=+167.1 avg_steps=58 epsilon=0.069 avg_loss=6.7 time=0m12s total=17m53s -[ep_9000] ep=8901-9000 avg_reward=+188.4 avg_steps=58 epsilon=0.067 avg_loss=6.6 time=0m12s total=18m05s -[ep_9100] ep=9001-9100 avg_reward=+225.5 avg_steps=70 epsilon=0.065 avg_loss=6.6 time=0m14s total=18m20s -[ep_9200] ep=9101-9200 avg_reward=+260.5 avg_steps=81 epsilon=0.063 avg_loss=6.5 time=0m17s total=18m37s -[ep_9300] ep=9201-9300 avg_reward=+250.0 avg_steps=77 epsilon=0.061 avg_loss=6.3 time=0m16s total=18m54s -[ep_9400] ep=9301-9400 avg_reward=+313.0 avg_steps=93 epsilon=0.060 avg_loss=6.1 time=0m19s total=19m14s -[ep_9500] ep=9401-9500 avg_reward=+314.2 avg_steps=93 epsilon=0.058 avg_loss=5.9 time=0m19s total=19m34s -[ep_9600] ep=9501-9600 avg_reward=+246.5 avg_steps=75 epsilon=0.056 avg_loss=5.7 time=0m16s total=19m50s -[ep_9700] ep=9601-9700 avg_reward=+274.8 avg_steps=82 epsilon=0.054 avg_loss=5.5 time=0m17s total=20m07s -[ep_9800] ep=9701-9800 avg_reward=+329.7 avg_steps=97 epsilon=0.053 avg_loss=5.5 time=0m20s total=20m28s -[ep_9900] ep=9801-9900 avg_reward=+286.5 avg_steps=87 epsilon=0.051 avg_loss=5.4 time=0m18s total=20m47s -[ep_10000] ep=9901-10000 avg_reward=+304.0 avg_steps=88 epsilon=0.050 avg_loss=5.5 time=0m18s total=21m05s -[ep_10100] ep=10001-10100 avg_reward=+349.1 avg_steps=99 epsilon=0.050 avg_loss=5.5 time=0m21s total=21m26s -[ep_10200] ep=10101-10200 avg_reward=+327.1 avg_steps=93 epsilon=0.050 avg_loss=5.3 time=0m19s total=21m46s -[ep_10300] ep=10201-10300 avg_reward=+347.8 avg_steps=98 epsilon=0.050 avg_loss=5.3 time=0m20s total=22m07s -[ep_10400] ep=10301-10400 avg_reward=+327.0 avg_steps=96 epsilon=0.050 avg_loss=5.1 time=0m20s total=22m28s -[ep_10500] ep=10401-10500 avg_reward=+330.2 avg_steps=96 epsilon=0.050 avg_loss=5.1 time=0m20s total=22m49s -[ep_10600] ep=10501-10600 avg_reward=+335.0 avg_steps=92 epsilon=0.050 avg_loss=5.0 time=0m19s total=23m09s -[ep_10700] ep=10601-10700 avg_reward=+313.9 avg_steps=88 epsilon=0.050 avg_loss=5.0 time=0m19s total=23m28s -[ep_10800] ep=10701-10800 avg_reward=+368.5 avg_steps=104 epsilon=0.050 avg_loss=5.0 time=0m22s total=23m50s -[ep_10900] ep=10801-10900 avg_reward=+366.0 avg_steps=104 epsilon=0.050 avg_loss=4.9 time=0m22s total=24m12s -[ep_11000] ep=10901-11000 avg_reward=+309.2 avg_steps=86 epsilon=0.050 avg_loss=4.9 time=0m18s total=24m31s -[ep_11100] ep=11001-11100 avg_reward=+354.0 avg_steps=100 epsilon=0.050 avg_loss=4.9 time=0m21s total=24m53s -[ep_11200] ep=11101-11200 avg_reward=+305.9 avg_steps=84 epsilon=0.050 avg_loss=4.9 time=0m18s total=25m11s -[ep_11300] ep=11201-11300 avg_reward=+304.6 avg_steps=87 epsilon=0.050 avg_loss=4.9 time=0m18s total=25m30s -[ep_11400] ep=11301-11400 avg_reward=+368.9 avg_steps=101 epsilon=0.050 avg_loss=4.8 time=0m21s total=25m52s -[ep_11500] ep=11401-11500 avg_reward=+342.0 avg_steps=93 epsilon=0.050 avg_loss=4.7 time=0m20s total=26m12s -[ep_11600] ep=11501-11600 avg_reward=+304.7 avg_steps=86 epsilon=0.050 avg_loss=4.7 time=0m18s total=26m30s -[ep_11700] ep=11601-11700 avg_reward=+354.6 avg_steps=99 epsilon=0.050 avg_loss=4.7 time=0m21s total=26m52s -[ep_11800] ep=11701-11800 avg_reward=+308.3 avg_steps=89 epsilon=0.050 avg_loss=4.7 time=0m19s total=27m11s -[ep_11900] ep=11801-11900 avg_reward=+321.1 avg_steps=93 epsilon=0.050 avg_loss=4.6 time=0m20s total=27m31s -[ep_12000] ep=11901-12000 avg_reward=+334.3 avg_steps=97 epsilon=0.050 avg_loss=4.7 time=0m20s total=27m52s -[ep_12100] ep=12001-12100 avg_reward=+367.0 avg_steps=104 epsilon=0.050 avg_loss=4.6 time=0m22s total=28m15s -[ep_12200] ep=12101-12200 avg_reward=+346.9 avg_steps=103 epsilon=0.050 avg_loss=4.4 time=0m22s total=28m37s -[ep_12300] ep=12201-12300 avg_reward=+333.0 avg_steps=96 epsilon=0.050 avg_loss=4.5 time=0m20s total=28m58s -[ep_12400] ep=12301-12400 avg_reward=+341.2 avg_steps=98 epsilon=0.050 avg_loss=4.3 time=0m21s total=29m19s -[ep_12500] ep=12401-12500 avg_reward=+342.6 avg_steps=97 epsilon=0.050 avg_loss=4.4 time=0m21s total=29m40s -[ep_12600] ep=12501-12600 avg_reward=+334.0 avg_steps=92 epsilon=0.050 avg_loss=4.3 time=0m19s total=30m00s -[ep_12700] ep=12601-12700 avg_reward=+332.6 avg_steps=92 epsilon=0.050 avg_loss=4.4 time=0m20s total=30m20s -[ep_12800] ep=12701-12800 avg_reward=+343.0 avg_steps=96 epsilon=0.050 avg_loss=4.4 time=0m20s total=30m41s -[ep_12900] ep=12801-12900 avg_reward=+344.9 avg_steps=101 epsilon=0.050 avg_loss=4.5 time=0m21s total=31m02s -[ep_13000] ep=12901-13000 avg_reward=+292.4 avg_steps=83 epsilon=0.050 avg_loss=4.5 time=0m17s total=31m20s -[ep_13100] ep=13001-13100 avg_reward=+323.1 avg_steps=89 epsilon=0.050 avg_loss=4.6 time=0m19s total=31m39s -[ep_13200] ep=13101-13200 avg_reward=+310.8 avg_steps=86 epsilon=0.050 avg_loss=4.4 time=0m18s total=31m58s -[ep_13300] ep=13201-13300 avg_reward=+308.8 avg_steps=85 epsilon=0.050 avg_loss=4.4 time=0m18s total=32m16s -[ep_13400] ep=13301-13400 avg_reward=+395.9 avg_steps=111 epsilon=0.050 avg_loss=4.4 time=0m23s total=32m40s -[ep_13500] ep=13401-13500 avg_reward=+395.5 avg_steps=111 epsilon=0.050 avg_loss=4.3 time=0m23s total=33m03s -[ep_13600] ep=13501-13600 avg_reward=+384.9 avg_steps=107 epsilon=0.050 avg_loss=4.2 time=0m23s total=33m26s -[ep_13700] ep=13601-13700 avg_reward=+342.1 avg_steps=96 epsilon=0.050 avg_loss=4.3 time=0m20s total=33m47s -[ep_13800] ep=13701-13800 avg_reward=+319.3 avg_steps=92 epsilon=0.050 avg_loss=4.3 time=0m21s total=34m09s -[ep_13900] ep=13801-13900 avg_reward=+337.6 avg_steps=98 epsilon=0.050 avg_loss=4.2 time=0m25s total=34m35s -[ep_14000] ep=13901-14000 avg_reward=+302.8 avg_steps=85 epsilon=0.050 avg_loss=4.3 time=0m18s total=34m53s -[ep_14100] ep=14001-14100 avg_reward=+337.7 avg_steps=94 epsilon=0.050 avg_loss=4.4 time=0m20s total=35m13s -[ep_14200] ep=14101-14200 avg_reward=+315.7 avg_steps=88 epsilon=0.050 avg_loss=4.2 time=0m18s total=35m32s -[ep_14300] ep=14201-14300 avg_reward=+337.8 avg_steps=99 epsilon=0.050 avg_loss=4.3 time=0m21s total=35m53s -[ep_14400] ep=14301-14400 avg_reward=+313.3 avg_steps=88 epsilon=0.050 avg_loss=4.3 time=0m18s total=36m12s -[ep_14500] ep=14401-14500 avg_reward=+322.5 avg_steps=91 epsilon=0.050 avg_loss=4.3 time=0m19s total=36m32s -[ep_14600] ep=14501-14600 avg_reward=+385.1 avg_steps=106 epsilon=0.050 avg_loss=4.2 time=0m22s total=36m55s -[ep_14700] ep=14601-14700 avg_reward=+333.2 avg_steps=92 epsilon=0.050 avg_loss=4.1 time=0m19s total=37m14s -[ep_14800] ep=14701-14800 avg_reward=+325.7 avg_steps=92 epsilon=0.050 avg_loss=4.0 time=0m19s total=37m34s -[ep_14900] ep=14801-14900 avg_reward=+308.1 avg_steps=87 epsilon=0.050 avg_loss=4.2 time=0m18s total=37m53s -[ep_15000] ep=14901-15000 avg_reward=+374.5 avg_steps=103 epsilon=0.050 avg_loss=4.2 time=0m22s total=38m15s -[ep_15100] ep=15001-15100 avg_reward=+334.0 avg_steps=94 epsilon=0.050 avg_loss=4.3 time=0m20s total=38m35s -[ep_15200] ep=15101-15200 avg_reward=+363.1 avg_steps=97 epsilon=0.050 avg_loss=4.4 time=0m20s total=38m56s -[ep_15300] ep=15201-15300 avg_reward=+367.9 avg_steps=104 epsilon=0.050 avg_loss=4.5 time=0m22s total=39m18s -[ep_15400] ep=15301-15400 avg_reward=+367.8 avg_steps=99 epsilon=0.050 avg_loss=4.4 time=0m21s total=39m39s -[ep_15500] ep=15401-15500 avg_reward=+331.6 avg_steps=93 epsilon=0.050 avg_loss=4.2 time=0m19s total=39m59s -[ep_15600] ep=15501-15600 avg_reward=+349.4 avg_steps=97 epsilon=0.050 avg_loss=4.1 time=0m20s total=40m20s -[ep_15700] ep=15601-15700 avg_reward=+374.1 avg_steps=108 epsilon=0.050 avg_loss=4.1 time=0m23s total=40m43s -[ep_15800] ep=15701-15800 avg_reward=+345.4 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=41m04s -[ep_15900] ep=15801-15900 avg_reward=+372.3 avg_steps=108 epsilon=0.050 avg_loss=4.1 time=0m23s total=41m27s -[ep_16000] ep=15901-16000 avg_reward=+351.0 avg_steps=95 epsilon=0.050 avg_loss=4.2 time=0m20s total=41m47s -[ep_16100] ep=16001-16100 avg_reward=+345.8 avg_steps=99 epsilon=0.050 avg_loss=4.2 time=0m21s total=42m08s -[ep_16200] ep=16101-16200 avg_reward=+403.9 avg_steps=110 epsilon=0.050 avg_loss=4.2 time=0m23s total=42m32s -[ep_16300] ep=16201-16300 avg_reward=+424.1 avg_steps=116 epsilon=0.050 avg_loss=4.2 time=0m24s total=42m57s -[ep_16400] ep=16301-16400 avg_reward=+383.7 avg_steps=104 epsilon=0.050 avg_loss=4.3 time=0m22s total=43m19s -[ep_16500] ep=16401-16500 avg_reward=+356.4 avg_steps=97 epsilon=0.050 avg_loss=4.1 time=0m20s total=43m40s -[ep_16600] ep=16501-16600 avg_reward=+367.3 avg_steps=104 epsilon=0.050 avg_loss=4.1 time=0m22s total=44m02s -[ep_16700] ep=16601-16700 avg_reward=+322.2 avg_steps=96 epsilon=0.050 avg_loss=4.0 time=0m20s total=44m23s -[ep_16800] ep=16701-16800 avg_reward=+376.6 avg_steps=103 epsilon=0.050 avg_loss=4.1 time=0m22s total=44m45s -[ep_16900] ep=16801-16900 avg_reward=+340.6 avg_steps=96 epsilon=0.050 avg_loss=4.1 time=0m20s total=45m06s -[ep_17000] ep=16901-17000 avg_reward=+353.4 avg_steps=99 epsilon=0.050 avg_loss=4.0 time=0m21s total=45m27s -[ep_17100] ep=17001-17100 avg_reward=+304.6 avg_steps=87 epsilon=0.050 avg_loss=4.0 time=0m18s total=45m46s -[ep_17200] ep=17101-17200 avg_reward=+322.3 avg_steps=90 epsilon=0.050 avg_loss=3.9 time=0m19s total=46m05s -[ep_17300] ep=17201-17300 avg_reward=+376.1 avg_steps=105 epsilon=0.050 avg_loss=4.0 time=0m22s total=46m28s -[ep_17400] ep=17301-17400 avg_reward=+333.3 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=46m48s -[ep_17500] ep=17401-17500 avg_reward=+363.4 avg_steps=104 epsilon=0.050 avg_loss=4.0 time=0m22s total=47m10s -[ep_17600] ep=17501-17600 avg_reward=+336.3 avg_steps=93 epsilon=0.050 avg_loss=3.9 time=0m20s total=47m30s -[ep_17700] ep=17601-17700 avg_reward=+347.2 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=47m51s -[ep_17800] ep=17701-17800 avg_reward=+344.4 avg_steps=98 epsilon=0.050 avg_loss=4.0 time=0m21s total=48m12s -[ep_17900] ep=17801-17900 avg_reward=+371.6 avg_steps=109 epsilon=0.050 avg_loss=4.0 time=0m23s total=48m35s -[ep_18000] ep=17901-18000 avg_reward=+433.1 avg_steps=119 epsilon=0.050 avg_loss=4.1 time=0m25s total=49m00s -[ep_18100] ep=18001-18100 avg_reward=+317.4 avg_steps=90 epsilon=0.050 avg_loss=4.2 time=0m19s total=49m20s -[ep_18200] ep=18101-18200 avg_reward=+381.8 avg_steps=106 epsilon=0.050 avg_loss=4.2 time=0m22s total=49m42s -[ep_18300] ep=18201-18300 avg_reward=+377.0 avg_steps=105 epsilon=0.050 avg_loss=4.1 time=0m22s total=50m05s -[ep_18400] ep=18301-18400 avg_reward=+377.0 avg_steps=101 epsilon=0.050 avg_loss=4.0 time=0m21s total=50m26s -[ep_18500] ep=18401-18500 avg_reward=+356.1 avg_steps=101 epsilon=0.050 avg_loss=4.0 time=0m21s total=50m48s -[ep_18600] ep=18501-18600 avg_reward=+409.6 avg_steps=109 epsilon=0.050 avg_loss=4.1 time=0m23s total=51m11s -[ep_18700] ep=18601-18700 avg_reward=+338.2 avg_steps=94 epsilon=0.050 avg_loss=4.1 time=0m20s total=51m31s -[ep_18800] ep=18701-18800 avg_reward=+354.6 avg_steps=104 epsilon=0.050 avg_loss=4.2 time=0m22s total=51m53s -[ep_18900] ep=18801-18900 avg_reward=+333.9 avg_steps=94 epsilon=0.050 avg_loss=4.2 time=0m20s total=52m14s -[ep_19000] ep=18901-19000 avg_reward=+388.2 avg_steps=106 epsilon=0.050 avg_loss=4.4 time=0m22s total=52m36s -[ep_19100] ep=19001-19100 avg_reward=+368.2 avg_steps=100 epsilon=0.050 avg_loss=4.3 time=0m21s total=52m58s -[ep_19200] ep=19101-19200 avg_reward=+435.8 avg_steps=121 epsilon=0.050 avg_loss=4.2 time=0m25s total=53m24s -[ep_19300] ep=19201-19300 avg_reward=+345.8 avg_steps=99 epsilon=0.050 avg_loss=4.1 time=0m21s total=53m45s -[ep_19400] ep=19301-19400 avg_reward=+372.5 avg_steps=104 epsilon=0.050 avg_loss=4.1 time=0m22s total=54m07s -[ep_19500] ep=19401-19500 avg_reward=+374.1 avg_steps=103 epsilon=0.050 avg_loss=3.9 time=0m21s total=54m29s -[ep_19600] ep=19501-19600 avg_reward=+387.1 avg_steps=109 epsilon=0.050 avg_loss=3.9 time=0m23s total=54m53s -[ep_19700] ep=19601-19700 avg_reward=+360.1 avg_steps=99 epsilon=0.050 avg_loss=3.9 time=0m21s total=55m14s -[ep_19800] ep=19701-19800 avg_reward=+387.3 avg_steps=107 epsilon=0.050 avg_loss=4.0 time=0m22s total=55m37s -[ep_19900] ep=19801-19900 avg_reward=+363.4 avg_steps=98 epsilon=0.050 avg_loss=3.9 time=0m20s total=55m58s -[ep_20000] ep=19901-20000 avg_reward=+335.6 avg_steps=94 epsilon=0.050 avg_loss=4.0 time=0m19s total=56m17s diff --git a/snake_observation.py b/snake_observation.py deleted file mode 100644 index f858e87..0000000 --- a/snake_observation.py +++ /dev/null @@ -1,31 +0,0 @@ -"""A custom observation_function for retro.examples.snake. - -Crops the board to a window centered on the snake's head before encoding, -using egocentric_board()/encode_board() from retro_gamer.observation as -building blocks. This replaces retro_gamer's old built-in egocentric flags — -cropping is now just something an observation_function does for itself. - -Point a run's config.toml at this with: - - [metadata] - observation_function = "snake_observation:egocentric_observation" - board_size = [17, 17] # must match RADIUS below: 2*8+1 = 17 -""" -import numpy as np -from retro.views.headless import HeadlessView -from retro_gamer.observation import egocentric_board, encode_board, encode_state - -CHARACTER_SET = ["@", "*", ">", "<", "^", "v"] -RADIUS = 8 - - -def egocentric_observation(game): - """Board cropped to a (2*RADIUS+1)^2 window around the snake's head, plus apple_dx/apple_dy.""" - view = HeadlessView() - view.on_game_start(game) - view.render(game) - head = game.get_agent_by_name("Snake head") - cropped = egocentric_board(view.board_characters, head.position, RADIUS) - board_vec = encode_board(cropped, CHARACTER_SET).flatten() - extras = encode_state(game.state, ["apple_dx", "apple_dy"]) - return np.concatenate([board_vec, extras]) diff --git a/snake_training.md b/snake_training.md deleted file mode 100644 index 5e57ba4..0000000 --- a/snake_training.md +++ /dev/null @@ -1,144 +0,0 @@ -# Snake Training: Conceptual Questions - -Answer each question in the space provided. Use evidence from the training log -and your observations of the agent at different checkpoints to support your answers. - ---- - -## 1. Feature selection - -In the first training attempt, the agent received the full 32×16 game board as -its input (6 × 32 × 16 = 3,072 numbers). The agent could see every character on -the board, yet it never learned to reliably find the apple after 45,000 episodes. - -When we added `apple_dx` and `apple_dy` — two numbers that encode the direction -from the snake's head to the apple — performance improved dramatically within -hundreds of episodes. - -**Why didn't the board encoding help the agent find the apple? What did the two -new features provide that the board encoding could not?** - -*Your answer:* - ---- - -## 2. Dimensionality reduction - -In the full-board experiment, the agent processed 3,072 input values. When we -switched to the egocentric view (a 17×17 window centered on the snake's head), -the board input shrank to 17 × 17 × 6 = 1,734 values. - -**How many input values did the egocentric view save compared to the full board? -What is one thing the agent gained from this change, and one thing it lost?** - -*Your answer:* - ---- - -## 3. Exploration vs. exploitation - -With `epsilon_decay = 0.995`, epsilon falls from 1.0 to 0.05 by episode ~450. -With `epsilon_decay = 0.9997` (used in the final run), epsilon is still 0.55 at -episode 2,000. - -**Sketch a rough curve of epsilon over time for each setting. Why does slower -decay produce a better-trained agent, even though it means the agent takes more -random actions overall?** - -*Your answer:* - ---- - -## 4. Runaway loss - -In one intermediate experiment, the loss grew from around 35 to hundreds of -thousands within a few hundred episodes: - -``` -[ep_0300] avg_loss=48.7 avg_reward=+8.1 -[ep_0500] avg_loss=347 avg_reward=+12.4 -[ep_0700] avg_loss=4,102 avg_reward=+6.5 -[ep_1100] avg_loss=686,000 avg_reward=-3.1 -``` - -This happened because the learning algorithm was using MSE (mean squared error) -loss, which is *quadratic* — an error of size 2 produces a loss of 4, an error -of size 10 produces a loss of 100. - -**Describe the feedback loop that caused the loss to spiral upward. Why does -Huber loss (which is linear for large errors) break this cycle?** - -*Your answer:* - ---- - -## 5. Interpreting the training curve - -Look at `runs/snake/training.log`. The reward climbs, then dips, then climbs -again: - -``` -[ep_1300] avg_reward= +31.6 avg_steps=49 -[ep_2300] avg_reward= -3.8 avg_steps=87 -[ep_4000] avg_reward= +53.2 avg_steps=42 -[ep_20000] avg_reward=+335.6 avg_steps=94 -``` - -Notice that around episode 4,000, avg_steps dropped sharply (from ~87 to 42) -at the same time reward jumped. Then by episode 20,000, steps rose again while -reward kept climbing. - -**What do you think the agent was doing at each of these stages? Use the -avg_steps and avg_reward numbers to support your interpretation.** - -*Your answer:* - ---- - -## 6. Policy observation - -Run these commands to watch the agent at three checkpoints: - -``` -retro-gamer play runs/snake --checkpoint ep_1300 -retro-gamer play runs/snake --checkpoint ep_4000 -retro-gamer play runs/snake --checkpoint ep_20000 -``` - -**Describe the agent's behavior at each checkpoint. What has the agent learned -by episode 4,000 that it hadn't yet learned at episode 1,300? What does the -episode 20,000 agent do that the earlier agents do not?** - -*ep_1300:* - -*ep_4000:* - -*ep_20000:* - ---- - -## 7. CNN vs. MLP - -In the first attempt (full board, no explicit features), we used a CNN -(`spatial = true`). In the final run (egocentric board + explicit features), we -used an MLP (`spatial = false`). - -**Why might an MLP be a reasonable choice when using the egocentric view, even -though the input is still a 2D board? What does the CNN offer that the MLP does -not, and why is that less important with an egocentric observation?** - -*Your answer:* - ---- - -## 8. Hyperparameter comparison - -Suppose you ran two otherwise identical training experiments: -- Run A: `learning_rate = 0.001` -- Run B: `learning_rate = 0.0001` - -**Based on what you learned from the runaway loss in Question 4, predict what -would happen in each run. What does this tell you about the trade-off when -choosing a learning rate?** - -*Your answer:* diff --git a/training_log.md b/training_log.md deleted file mode 100644 index 521b65a..0000000 --- a/training_log.md +++ /dev/null @@ -1,103 +0,0 @@ -# Forager Training Log - -Document each training attempt below. For each attempt, write your hypothesis -before you run the experiment, then fill in the evidence and analysis after. - -Use `retro-gamer info runs/forager/` to see a summary of your run, -`cat runs/forager/training.log` to see the full log, and -`retro-gamer plot runs/forager/ -o runs/forager/training.png` to graph it. - ---- - -## Attempt 1 - -### Hypothesis - -*Before training, predict what will happen with the default configuration. -Will the agent learn to find the food? How quickly? What might go wrong?* - -Your prediction: - -### Configuration - -*Copy the relevant sections of `runs/forager/config.toml` here.* - -```toml - -``` - -### Evidence - -*Paste the first and last few lines of your training log, and any interesting -moments in between.* - -``` - -``` - -### Analysis - -*What happened? How do the numbers — avg_reward, avg_steps, epsilon, avg_loss — -tell the story of what the agent learned? Did the result match your prediction?* - ---- - -## Attempt 2 - -### Hypothesis - -*Based on what you observed in Attempt 1, what will you change and why? -Predict the outcome.* - -### Configuration - -```toml - -``` - -### Evidence - -``` - -``` - -### Analysis - ---- - -## Attempt 3 (if needed) - -### Hypothesis - -### Configuration - -```toml - -``` - -### Evidence - -``` - -``` - -### Analysis - ---- - -## Final analysis - -**Which attempt produced the best-trained agent? Run `retro-gamer play` on your -best run's checkpoints and describe what the agent does.** - -*Your answer:* - -**Compare two of your attempts. What changed between them, and how did that -change affect the training curve?** - -*Your answer:* - -**If you had more time, what would you try next to improve the agent further? -Refer to specific hyperparameters or configuration options.** - -*Your answer:* diff --git a/uv.lock b/uv.lock index c440a25..531c1f0 100644 --- a/uv.lock +++ b/uv.lock @@ -424,8 +424,8 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "retro-gamer", specifier = ">=0.2.0" }, - { name = "retro-games", specifier = ">=2.4.0" }, + { name = "retro-gamer", editable = "../../packages/retro-gamer" }, + { name = "retro-games", specifier = ">=2.5.0" }, ] [[package]] @@ -1069,8 +1069,8 @@ wheels = [ [[package]] name = "retro-gamer" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } +version = "0.3.4" +source = { editable = "../../packages/retro-gamer" } dependencies = [ { name = "click" }, { name = "matplotlib" }, @@ -1083,21 +1083,41 @@ dependencies = [ { name = "torch" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/22/a861bc238bb50a016bbe184ad909ffabffad605869f4ff829b17d32b8cd1/retro_gamer-0.2.0.tar.gz", hash = "sha256:44f3fee63aef6847b686fe4e32983c160012d59fa0563c94bdcdbb13d40c044d", size = 154128, upload-time = "2026-06-24T00:46:05.615Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/9e/bf7fb5e607cafcb2bca320c10f2670a6edb37bdc80358fbf671bf51636e3/retro_gamer-0.2.0-py3-none-any.whl", hash = "sha256:8fc91e6c8d7709268ac97e362182d40d4d440046e4b36604413c53039a2e4713", size = 27840, upload-time = "2026-06-24T00:46:04.535Z" }, + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.0" }, + { name = "matplotlib", specifier = ">=3.7" }, + { name = "numpy", specifier = ">=1.24" }, + { name = "plotext", specifier = ">=5.0" }, + { name = "retro-games", editable = "../../packages/retro" }, + { name = "seaborn", specifier = ">=0.13" }, + { name = "tomli-w", specifier = ">=1.0" }, + { name = "torch", specifier = ">=2.0" }, + { name = "tqdm", specifier = ">=4.0" }, +] + +[package.metadata.requires-dev] +documentation = [ + { name = "sphinx", specifier = ">=7.0" }, + { name = "sphinx-rtd-theme", specifier = ">=2.0" }, ] [[package]] name = "retro-games" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } +version = "2.5.0" +source = { editable = "../../packages/retro" } dependencies = [ { name = "blessed" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/07/caafc47b5f40a0198467e0d994c6433b91c84c29cd235cf794dee4381ae7/retro_games-2.4.0.tar.gz", hash = "sha256:6a4d988afa922da69402b169b82e4136a18988f8d34b1740ea4f23382ab4ab35", size = 21723, upload-time = "2026-06-22T20:44:42.112Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/7c/e97207d25668790fd6a246be48beef798c8b41accd7c8c61026e3dfa9c62/retro_games-2.4.0-py3-none-any.whl", hash = "sha256:ea15765ea639d2284937dd0f32475c5ea6a9a9da1b54d09f35a2e8043efea2ba", size = 34830, upload-time = "2026-06-22T20:44:42.964Z" }, + +[package.metadata] +requires-dist = [{ name = "blessed", specifier = ">=1.33.0" }] + +[package.metadata.requires-dev] +documentation = [ + { name = "sphinx", specifier = ">=8.1.3" }, + { name = "sphinx-rtd-theme", specifier = ">=3.0" }, ] [[package]]