e8a24ae7be
q_learning.py mixed a bespoke BabySnake environment wrapper, the two functions students implement, a training loop, and a terminal watch routine in one file, with no tests and no single command to run training. - q_learning.py / q_learning_solution.py now hold only choose_action and update_q, with actions as an explicit parameter instead of a module-global, so the file has no babysnake/retro/retro_gamer imports at all. - train_babysnake.py builds GameEnvironment directly from babysnake's pyproject metadata, trains, and watches the trained agent in one command: `python train_babysnake.py`. It also caps steps per episode, since a lucky random walk that keeps finding food can otherwise make an episode run unboundedly long. - test_q_learning.py adds unittest coverage for both functions with no game dependency. - questions.md adds a checkpoint instructing students to get the tests passing before training, and points the post-training step at train_babysnake.py.
63 lines
2.5 KiB
Python
63 lines
2.5 KiB
Python
# test_q_learning.py
|
|
# ------------
|
|
# Defines tests for `q_learning`. Run this program with `python test_q_learning.py`.
|
|
# You don't need to edit this file.
|
|
#
|
|
# Get every test here passing before you run train_babysnake.py — errors are
|
|
# much easier to spot here than during training.
|
|
|
|
from unittest import TestCase, main
|
|
import random
|
|
|
|
from q_learning import choose_action, update_q
|
|
|
|
ACTIONS = ["UP", "DOWN", "LEFT", "RIGHT"]
|
|
|
|
|
|
class TestChooseAction(TestCase):
|
|
def test_greedy_picks_highest_q_value(self):
|
|
q_table = {("s", "UP"): 1.0, ("s", "DOWN"): 5.0, ("s", "LEFT"): 2.0, ("s", "RIGHT"): 0.0}
|
|
self.assertEqual(choose_action(q_table, "s", ACTIONS, epsilon=0.0), "DOWN")
|
|
|
|
def test_unseen_state_defaults_to_zero_and_picks_first_action(self):
|
|
self.assertEqual(choose_action({}, "new_state", ACTIONS, epsilon=0.0), ACTIONS[0])
|
|
|
|
def test_fully_random_explores_more_than_one_action(self):
|
|
random.seed(0)
|
|
q_table = {("s", "UP"): 100.0} # UP is clearly the best action
|
|
results = {choose_action(q_table, "s", ACTIONS, epsilon=1.0) for _ in range(50)}
|
|
self.assertGreater(len(results), 1)
|
|
|
|
def test_always_returns_a_valid_action(self):
|
|
for _ in range(20):
|
|
result = choose_action({}, "s", ACTIONS, epsilon=0.5)
|
|
self.assertIn(result, ACTIONS)
|
|
|
|
|
|
class TestUpdateQ(TestCase):
|
|
def test_basic_bellman_update(self):
|
|
q_table = {("s", "UP"): 0.0}
|
|
update_q(q_table, "s", "UP", reward=1.0, next_state="t", actions=ACTIONS, alpha=0.5, gamma=0.9)
|
|
# old_q=0, best_next_q=0 (unseen) -> target=1.0, new_q = 0 + 0.5*(1.0-0) = 0.5
|
|
self.assertAlmostEqual(q_table[("s", "UP")], 0.5)
|
|
|
|
def test_uses_best_next_q_value(self):
|
|
q_table = {("s", "UP"): 0.0, ("t", "UP"): 2.0, ("t", "DOWN"): 5.0}
|
|
update_q(q_table, "s", "UP", reward=0.0, next_state="t", actions=ACTIONS, alpha=1.0, gamma=1.0)
|
|
# target = 0 + 1.0*5.0 = 5.0; alpha=1 fully replaces the old value
|
|
self.assertAlmostEqual(q_table[("s", "UP")], 5.0)
|
|
|
|
def test_alpha_zero_means_no_change(self):
|
|
q_table = {("s", "UP"): 3.0}
|
|
update_q(q_table, "s", "UP", reward=10.0, next_state="t", actions=ACTIONS, alpha=0.0, gamma=0.9)
|
|
self.assertAlmostEqual(q_table[("s", "UP")], 3.0)
|
|
|
|
def test_only_updates_the_given_state_action_pair(self):
|
|
q_table = {}
|
|
update_q(q_table, "s", "UP", reward=1.0, next_state="t", actions=ACTIONS, alpha=0.5, gamma=0.9)
|
|
self.assertEqual(set(q_table.keys()), {("s", "UP")})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|