diff --git a/.gitignore b/.gitignore index e636fc3..850b9ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,11 @@ **/__pycache__/* runs/* +!runs/snake/ +!runs/snake/config.toml +!runs/snake/training.log +!runs/snake/checkpoints/ +runs/snake/checkpoints/* +!runs/snake/checkpoints/ep_1300.pt +!runs/snake/checkpoints/ep_2300.pt +!runs/snake/checkpoints/ep_4000.pt +!runs/snake/checkpoints/ep_20000.pt diff --git a/runs/snake/checkpoints/ep_1300.pt b/runs/snake/checkpoints/ep_1300.pt new file mode 100644 index 0000000..88192e3 Binary files /dev/null and b/runs/snake/checkpoints/ep_1300.pt differ diff --git a/runs/snake/checkpoints/ep_20000.pt b/runs/snake/checkpoints/ep_20000.pt new file mode 100644 index 0000000..e32ecc3 Binary files /dev/null and b/runs/snake/checkpoints/ep_20000.pt differ diff --git a/runs/snake/checkpoints/ep_2300.pt b/runs/snake/checkpoints/ep_2300.pt new file mode 100644 index 0000000..4929f6b Binary files /dev/null and b/runs/snake/checkpoints/ep_2300.pt differ diff --git a/runs/snake/checkpoints/ep_4000.pt b/runs/snake/checkpoints/ep_4000.pt new file mode 100644 index 0000000..9e92e38 Binary files /dev/null and b/runs/snake/checkpoints/ep_4000.pt differ diff --git a/runs/snake/config.toml b/runs/snake/config.toml new file mode 100644 index 0000000..3bebc68 --- /dev/null +++ b/runs/snake/config.toml @@ -0,0 +1,53 @@ +[game] +module = "retro.examples.snake" + +[metadata] +actions = [ + "KEY_RIGHT", + "KEY_UP", + "KEY_LEFT", + "KEY_DOWN", +] +reward = "reward" +extras_size = 2 +board_size = [ + 17, + 17, +] +character_set = [ + "@", + "*", + ">", + "<", + "^", + "v", +] +observation_function = "snake_observation:egocentric_observation" + +[preprocessing] +spatial = false +board = true +observe_state = [] + +[model] +hidden_sizes = [ + 128, + 64, +] + +[training] +learning_rate = 0.0001 +learning_rate_decay = 0.9999 +gamma = 0.99 +epsilon = 1.0 +epsilon_decay = 0.9997 +epsilon_min = 0.05 +batch_size = 64 +memory_capacity = 50000 +target_update_freq = 500 +train_every = 4 +training_episodes = 20000 +prioritize_experiences = true +exploration_turns = 200 +unknown_character_strategy = "ignore" +max_turns_per_episode = 2000 diff --git a/runs/snake/training.log b/runs/snake/training.log new file mode 100644 index 0000000..f8878b0 --- /dev/null +++ b/runs/snake/training.log @@ -0,0 +1,213 @@ +[INIT] === Network Architecture === +[INIT] Board: 17×17, character set: 6 chars (one-hot per cell) +[INIT] Observed state features: 2 | Actions (incl. no-op): 5 +[INIT] spatial=False → using MLP architecture +[INIT] Rationale: the board encodes UI/status rather than a spatial scene; +[INIT] a flat MLP over the full observation is sufficient. +[INIT] MLP: 1736 → 128 → 64 → 5 +[INIT] Hidden layers: 2 | Layer sizes: [128, 64] +[INIT] Output: 5 Q-values +[INIT] Actions: ['KEY_RIGHT', 'KEY_UP', 'KEY_LEFT', 'KEY_DOWN'] + (no-op) +[INIT] Device: mps + +=== Training started | 2026-06-23 22:39:07 === +[ep_0100] ep=0001-0100 avg_reward=-7.8 avg_steps=49 epsilon=0.970 avg_loss=0.8 time=0m06s total=0m06s +[ep_0200] ep=0101-0200 avg_reward=-5.7 avg_steps=51 epsilon=0.942 avg_loss=0.9 time=0m07s total=0m13s +[ep_0300] ep=0201-0300 avg_reward=+0.1 avg_steps=49 epsilon=0.914 avg_loss=1.2 time=0m06s total=0m20s +[ep_0400] ep=0301-0400 avg_reward=+7.9 avg_steps=70 epsilon=0.887 avg_loss=1.6 time=0m09s total=0m30s +[ep_0500] ep=0401-0500 avg_reward=+12.8 avg_steps=57 epsilon=0.861 avg_loss=1.9 time=0m08s total=0m38s +[ep_0600] ep=0501-0600 avg_reward=+20.5 avg_steps=57 epsilon=0.835 avg_loss=2.3 time=0m08s total=0m47s +[ep_0700] ep=0601-0700 avg_reward=+23.7 avg_steps=62 epsilon=0.811 avg_loss=2.8 time=0m10s total=0m58s +[ep_0800] ep=0701-0800 avg_reward=+21.2 avg_steps=56 epsilon=0.787 avg_loss=3.4 time=0m10s total=1m08s +[ep_0900] ep=0801-0900 avg_reward=+30.1 avg_steps=55 epsilon=0.763 avg_loss=4.1 time=0m10s total=1m19s +[ep_1000] ep=0901-1000 avg_reward=+33.6 avg_steps=59 epsilon=0.741 avg_loss=4.6 time=0m11s total=1m31s +[ep_1100] ep=1001-1100 avg_reward=+36.0 avg_steps=58 epsilon=0.719 avg_loss=5.4 time=0m11s total=1m43s +[ep_1200] ep=1101-1200 avg_reward=+34.0 avg_steps=50 epsilon=0.698 avg_loss=6.1 time=0m10s total=1m53s +[ep_1300] ep=1201-1300 avg_reward=+31.6 avg_steps=49 epsilon=0.677 avg_loss=6.7 time=0m09s total=2m03s +[ep_1400] ep=1301-1400 avg_reward=+30.4 avg_steps=60 epsilon=0.657 avg_loss=7.2 time=0m12s total=2m15s +[ep_1500] ep=1401-1500 avg_reward=+28.0 avg_steps=65 epsilon=0.638 avg_loss=7.6 time=0m13s total=2m28s +[ep_1600] ep=1501-1600 avg_reward=+16.3 avg_steps=68 epsilon=0.619 avg_loss=7.6 time=0m14s total=2m42s +[ep_1700] ep=1601-1700 avg_reward=+10.1 avg_steps=76 epsilon=0.600 avg_loss=7.7 time=0m18s total=3m01s +[ep_1800] ep=1701-1800 avg_reward=+6.9 avg_steps=89 epsilon=0.583 avg_loss=7.6 time=0m20s total=3m21s +[ep_1900] ep=1801-1900 avg_reward=+1.9 avg_steps=85 epsilon=0.565 avg_loss=7.4 time=0m16s total=3m38s +[ep_2000] ep=1901-2000 avg_reward=+1.1 avg_steps=93 epsilon=0.549 avg_loss=6.9 time=0m18s total=3m57s +[ep_2100] ep=2001-2100 avg_reward=+1.4 avg_steps=105 epsilon=0.533 avg_loss=6.2 time=0m21s total=4m18s +[ep_2200] ep=2101-2200 avg_reward=-0.8 avg_steps=83 epsilon=0.517 avg_loss=5.5 time=0m16s total=4m34s +[ep_2300] ep=2201-2300 avg_reward=-3.8 avg_steps=87 epsilon=0.502 avg_loss=5.1 time=0m17s total=4m52s +[ep_2400] ep=2301-2400 avg_reward=-0.1 avg_steps=89 epsilon=0.487 avg_loss=4.6 time=0m17s total=5m09s +[ep_2500] ep=2401-2500 avg_reward=-2.5 avg_steps=94 epsilon=0.472 avg_loss=4.1 time=0m18s total=5m28s +[ep_2600] ep=2501-2600 avg_reward=+1.2 avg_steps=97 epsilon=0.458 avg_loss=3.7 time=0m19s total=5m48s +[ep_2700] ep=2601-2700 avg_reward=+8.0 avg_steps=85 epsilon=0.445 avg_loss=3.4 time=0m16s total=6m05s +[ep_2800] ep=2701-2800 avg_reward=+5.2 avg_steps=83 epsilon=0.432 avg_loss=2.9 time=0m16s total=6m21s +[ep_2900] ep=2801-2900 avg_reward=+8.4 avg_steps=89 epsilon=0.419 avg_loss=2.7 time=0m17s total=6m39s +[ep_3000] ep=2901-3000 avg_reward=+4.0 avg_steps=98 epsilon=0.407 avg_loss=2.4 time=0m19s total=6m59s +[ep_3100] ep=3001-3100 avg_reward=+6.0 avg_steps=80 epsilon=0.394 avg_loss=2.1 time=0m16s total=7m15s +[ep_3200] ep=3101-3200 avg_reward=+7.0 avg_steps=86 epsilon=0.383 avg_loss=2.0 time=0m17s total=7m33s +[ep_3300] ep=3201-3300 avg_reward=+4.3 avg_steps=93 epsilon=0.372 avg_loss=1.8 time=0m19s total=7m52s +[ep_3400] ep=3301-3400 avg_reward=+2.7 avg_steps=98 epsilon=0.361 avg_loss=1.8 time=0m20s total=8m12s +[ep_3500] ep=3401-3500 avg_reward=+7.7 avg_steps=99 epsilon=0.350 avg_loss=1.6 time=0m20s total=8m32s +[ep_3600] ep=3501-3600 avg_reward=+21.0 avg_steps=86 epsilon=0.340 avg_loss=1.5 time=0m17s total=8m50s +[ep_3700] ep=3601-3700 avg_reward=+30.8 avg_steps=67 epsilon=0.330 avg_loss=1.4 time=0m13s total=9m03s +[ep_3800] ep=3701-3800 avg_reward=+38.4 avg_steps=53 epsilon=0.320 avg_loss=1.3 time=0m10s total=9m14s +[ep_3900] ep=3801-3900 avg_reward=+48.0 avg_steps=49 epsilon=0.310 avg_loss=1.4 time=0m09s total=9m24s +[ep_4000] ep=3901-4000 avg_reward=+53.2 avg_steps=42 epsilon=0.301 avg_loss=1.4 time=0m08s total=9m33s +[ep_4100] ep=4001-4100 avg_reward=+49.1 avg_steps=47 epsilon=0.292 avg_loss=1.5 time=0m09s total=9m42s +[ep_4200] ep=4101-4200 avg_reward=+49.9 avg_steps=50 epsilon=0.284 avg_loss=1.6 time=0m10s total=9m53s +[ep_4300] ep=4201-4300 avg_reward=+54.5 avg_steps=48 epsilon=0.275 avg_loss=1.7 time=0m09s total=10m02s +[ep_4400] ep=4301-4400 avg_reward=+57.5 avg_steps=48 epsilon=0.267 avg_loss=1.8 time=0m09s total=10m12s +[ep_4500] ep=4401-4500 avg_reward=+53.6 avg_steps=47 epsilon=0.259 avg_loss=1.9 time=0m09s total=10m22s +[ep_4600] ep=4501-4600 avg_reward=+59.3 avg_steps=44 epsilon=0.252 avg_loss=1.9 time=0m09s total=10m31s +[ep_4700] ep=4601-4700 avg_reward=+60.2 avg_steps=39 epsilon=0.244 avg_loss=1.8 time=0m08s total=10m39s +[ep_4800] ep=4701-4800 avg_reward=+58.6 avg_steps=40 epsilon=0.237 avg_loss=1.8 time=0m08s total=10m47s +[ep_4900] ep=4801-4900 avg_reward=+64.5 avg_steps=54 epsilon=0.230 avg_loss=1.9 time=0m11s total=10m59s +[ep_5000] ep=4901-5000 avg_reward=+72.9 avg_steps=37 epsilon=0.223 avg_loss=1.9 time=0m07s total=11m06s +[ep_5100] ep=5001-5100 avg_reward=+72.2 avg_steps=36 epsilon=0.216 avg_loss=1.9 time=0m07s total=11m14s +[ep_5200] ep=5101-5200 avg_reward=+71.6 avg_steps=39 epsilon=0.210 avg_loss=1.9 time=0m08s total=11m22s +[ep_5300] ep=5201-5300 avg_reward=+67.9 avg_steps=34 epsilon=0.204 avg_loss=1.9 time=0m07s total=11m29s +[ep_5400] ep=5301-5400 avg_reward=+89.2 avg_steps=43 epsilon=0.198 avg_loss=1.9 time=0m09s total=11m38s +[ep_5500] ep=5401-5500 avg_reward=+92.0 avg_steps=42 epsilon=0.192 avg_loss=2.0 time=0m08s total=11m47s +[ep_5600] ep=5501-5600 avg_reward=+86.4 avg_steps=35 epsilon=0.186 avg_loss=2.1 time=0m07s total=11m55s +[ep_5700] ep=5601-5700 avg_reward=+92.5 avg_steps=37 epsilon=0.181 avg_loss=2.2 time=0m07s total=12m02s +[ep_5800] ep=5701-5800 avg_reward=+98.5 avg_steps=45 epsilon=0.175 avg_loss=2.3 time=0m09s total=12m12s +[ep_5900] ep=5801-5900 avg_reward=+100.9 avg_steps=39 epsilon=0.170 avg_loss=2.5 time=0m08s total=12m20s +[ep_6000] ep=5901-6000 avg_reward=+95.3 avg_steps=40 epsilon=0.165 avg_loss=2.7 time=0m08s total=12m28s +[ep_6100] ep=6001-6100 avg_reward=+92.4 avg_steps=40 epsilon=0.160 avg_loss=2.9 time=0m08s total=12m37s +[ep_6200] ep=6101-6200 avg_reward=+100.2 avg_steps=45 epsilon=0.156 avg_loss=3.1 time=0m09s total=12m46s +[ep_6300] ep=6201-6300 avg_reward=+96.4 avg_steps=43 epsilon=0.151 avg_loss=3.5 time=0m09s total=12m56s +[ep_6400] ep=6301-6400 avg_reward=+107.5 avg_steps=45 epsilon=0.147 avg_loss=3.9 time=0m09s total=13m05s +[ep_6500] ep=6401-6500 avg_reward=+87.4 avg_steps=38 epsilon=0.142 avg_loss=4.1 time=0m08s total=13m13s +[ep_6600] ep=6501-6600 avg_reward=+126.1 avg_steps=51 epsilon=0.138 avg_loss=4.7 time=0m10s total=13m24s +[ep_6700] ep=6601-6700 avg_reward=+118.6 avg_steps=42 epsilon=0.134 avg_loss=5.1 time=0m08s total=13m33s +[ep_6800] ep=6701-6800 avg_reward=+117.7 avg_steps=44 epsilon=0.130 avg_loss=5.7 time=0m09s total=13m42s +[ep_6900] ep=6801-6900 avg_reward=+154.6 avg_steps=56 epsilon=0.126 avg_loss=6.2 time=0m11s total=13m54s +[ep_7000] ep=6901-7000 avg_reward=+129.3 avg_steps=44 epsilon=0.122 avg_loss=6.8 time=0m09s total=14m04s +[ep_7100] ep=7001-7100 avg_reward=+114.2 avg_steps=40 epsilon=0.119 avg_loss=7.2 time=0m08s total=14m12s +[ep_7200] ep=7101-7200 avg_reward=+147.2 avg_steps=52 epsilon=0.115 avg_loss=7.6 time=0m11s total=14m24s +[ep_7300] ep=7201-7300 avg_reward=+122.9 avg_steps=45 epsilon=0.112 avg_loss=8.1 time=0m09s total=14m33s +[ep_7400] ep=7301-7400 avg_reward=+136.2 avg_steps=47 epsilon=0.109 avg_loss=8.4 time=0m10s total=14m43s +[ep_7500] ep=7401-7500 avg_reward=+151.2 avg_steps=51 epsilon=0.105 avg_loss=8.5 time=0m11s total=14m54s +[ep_7600] ep=7501-7600 avg_reward=+150.5 avg_steps=56 epsilon=0.102 avg_loss=8.5 time=0m11s total=15m06s +[ep_7700] ep=7601-7700 avg_reward=+120.3 avg_steps=46 epsilon=0.099 avg_loss=8.6 time=0m09s total=15m16s +[ep_7800] ep=7701-7800 avg_reward=+121.5 avg_steps=47 epsilon=0.096 avg_loss=8.5 time=0m09s total=15m26s +[ep_7900] ep=7801-7900 avg_reward=+160.6 avg_steps=60 epsilon=0.093 avg_loss=8.7 time=0m12s total=15m39s +[ep_8000] ep=7901-8000 avg_reward=+134.2 avg_steps=53 epsilon=0.091 avg_loss=8.6 time=0m11s total=15m50s +[ep_8100] ep=8001-8100 avg_reward=+141.0 avg_steps=51 epsilon=0.088 avg_loss=8.5 time=0m10s total=16m01s +[ep_8200] ep=8101-8200 avg_reward=+167.7 avg_steps=57 epsilon=0.085 avg_loss=8.2 time=0m12s total=16m13s +[ep_8300] ep=8201-8300 avg_reward=+165.9 avg_steps=56 epsilon=0.083 avg_loss=8.0 time=0m11s total=16m25s +[ep_8400] ep=8301-8400 avg_reward=+199.9 avg_steps=71 epsilon=0.080 avg_loss=7.9 time=0m15s total=16m40s +[ep_8500] ep=8401-8500 avg_reward=+196.7 avg_steps=66 epsilon=0.078 avg_loss=7.5 time=0m14s total=16m54s +[ep_8600] ep=8501-8600 avg_reward=+218.4 avg_steps=71 epsilon=0.076 avg_loss=7.4 time=0m15s total=17m10s +[ep_8700] ep=8601-8700 avg_reward=+204.1 avg_steps=74 epsilon=0.074 avg_loss=7.1 time=0m15s total=17m25s +[ep_8800] ep=8701-8800 avg_reward=+216.2 avg_steps=70 epsilon=0.071 avg_loss=6.8 time=0m14s total=17m40s +[ep_8900] ep=8801-8900 avg_reward=+167.1 avg_steps=58 epsilon=0.069 avg_loss=6.7 time=0m12s total=17m53s +[ep_9000] ep=8901-9000 avg_reward=+188.4 avg_steps=58 epsilon=0.067 avg_loss=6.6 time=0m12s total=18m05s +[ep_9100] ep=9001-9100 avg_reward=+225.5 avg_steps=70 epsilon=0.065 avg_loss=6.6 time=0m14s total=18m20s +[ep_9200] ep=9101-9200 avg_reward=+260.5 avg_steps=81 epsilon=0.063 avg_loss=6.5 time=0m17s total=18m37s +[ep_9300] ep=9201-9300 avg_reward=+250.0 avg_steps=77 epsilon=0.061 avg_loss=6.3 time=0m16s total=18m54s +[ep_9400] ep=9301-9400 avg_reward=+313.0 avg_steps=93 epsilon=0.060 avg_loss=6.1 time=0m19s total=19m14s +[ep_9500] ep=9401-9500 avg_reward=+314.2 avg_steps=93 epsilon=0.058 avg_loss=5.9 time=0m19s total=19m34s +[ep_9600] ep=9501-9600 avg_reward=+246.5 avg_steps=75 epsilon=0.056 avg_loss=5.7 time=0m16s total=19m50s +[ep_9700] ep=9601-9700 avg_reward=+274.8 avg_steps=82 epsilon=0.054 avg_loss=5.5 time=0m17s total=20m07s +[ep_9800] ep=9701-9800 avg_reward=+329.7 avg_steps=97 epsilon=0.053 avg_loss=5.5 time=0m20s total=20m28s +[ep_9900] ep=9801-9900 avg_reward=+286.5 avg_steps=87 epsilon=0.051 avg_loss=5.4 time=0m18s total=20m47s +[ep_10000] ep=9901-10000 avg_reward=+304.0 avg_steps=88 epsilon=0.050 avg_loss=5.5 time=0m18s total=21m05s +[ep_10100] ep=10001-10100 avg_reward=+349.1 avg_steps=99 epsilon=0.050 avg_loss=5.5 time=0m21s total=21m26s +[ep_10200] ep=10101-10200 avg_reward=+327.1 avg_steps=93 epsilon=0.050 avg_loss=5.3 time=0m19s total=21m46s +[ep_10300] ep=10201-10300 avg_reward=+347.8 avg_steps=98 epsilon=0.050 avg_loss=5.3 time=0m20s total=22m07s +[ep_10400] ep=10301-10400 avg_reward=+327.0 avg_steps=96 epsilon=0.050 avg_loss=5.1 time=0m20s total=22m28s +[ep_10500] ep=10401-10500 avg_reward=+330.2 avg_steps=96 epsilon=0.050 avg_loss=5.1 time=0m20s total=22m49s +[ep_10600] ep=10501-10600 avg_reward=+335.0 avg_steps=92 epsilon=0.050 avg_loss=5.0 time=0m19s total=23m09s +[ep_10700] ep=10601-10700 avg_reward=+313.9 avg_steps=88 epsilon=0.050 avg_loss=5.0 time=0m19s total=23m28s +[ep_10800] ep=10701-10800 avg_reward=+368.5 avg_steps=104 epsilon=0.050 avg_loss=5.0 time=0m22s total=23m50s +[ep_10900] ep=10801-10900 avg_reward=+366.0 avg_steps=104 epsilon=0.050 avg_loss=4.9 time=0m22s total=24m12s +[ep_11000] ep=10901-11000 avg_reward=+309.2 avg_steps=86 epsilon=0.050 avg_loss=4.9 time=0m18s total=24m31s +[ep_11100] ep=11001-11100 avg_reward=+354.0 avg_steps=100 epsilon=0.050 avg_loss=4.9 time=0m21s total=24m53s +[ep_11200] ep=11101-11200 avg_reward=+305.9 avg_steps=84 epsilon=0.050 avg_loss=4.9 time=0m18s total=25m11s +[ep_11300] ep=11201-11300 avg_reward=+304.6 avg_steps=87 epsilon=0.050 avg_loss=4.9 time=0m18s total=25m30s +[ep_11400] ep=11301-11400 avg_reward=+368.9 avg_steps=101 epsilon=0.050 avg_loss=4.8 time=0m21s total=25m52s +[ep_11500] ep=11401-11500 avg_reward=+342.0 avg_steps=93 epsilon=0.050 avg_loss=4.7 time=0m20s total=26m12s +[ep_11600] ep=11501-11600 avg_reward=+304.7 avg_steps=86 epsilon=0.050 avg_loss=4.7 time=0m18s total=26m30s +[ep_11700] ep=11601-11700 avg_reward=+354.6 avg_steps=99 epsilon=0.050 avg_loss=4.7 time=0m21s total=26m52s +[ep_11800] ep=11701-11800 avg_reward=+308.3 avg_steps=89 epsilon=0.050 avg_loss=4.7 time=0m19s total=27m11s +[ep_11900] ep=11801-11900 avg_reward=+321.1 avg_steps=93 epsilon=0.050 avg_loss=4.6 time=0m20s total=27m31s +[ep_12000] ep=11901-12000 avg_reward=+334.3 avg_steps=97 epsilon=0.050 avg_loss=4.7 time=0m20s total=27m52s +[ep_12100] ep=12001-12100 avg_reward=+367.0 avg_steps=104 epsilon=0.050 avg_loss=4.6 time=0m22s total=28m15s +[ep_12200] ep=12101-12200 avg_reward=+346.9 avg_steps=103 epsilon=0.050 avg_loss=4.4 time=0m22s total=28m37s +[ep_12300] ep=12201-12300 avg_reward=+333.0 avg_steps=96 epsilon=0.050 avg_loss=4.5 time=0m20s total=28m58s +[ep_12400] ep=12301-12400 avg_reward=+341.2 avg_steps=98 epsilon=0.050 avg_loss=4.3 time=0m21s total=29m19s +[ep_12500] ep=12401-12500 avg_reward=+342.6 avg_steps=97 epsilon=0.050 avg_loss=4.4 time=0m21s total=29m40s +[ep_12600] ep=12501-12600 avg_reward=+334.0 avg_steps=92 epsilon=0.050 avg_loss=4.3 time=0m19s total=30m00s +[ep_12700] ep=12601-12700 avg_reward=+332.6 avg_steps=92 epsilon=0.050 avg_loss=4.4 time=0m20s total=30m20s +[ep_12800] ep=12701-12800 avg_reward=+343.0 avg_steps=96 epsilon=0.050 avg_loss=4.4 time=0m20s total=30m41s +[ep_12900] ep=12801-12900 avg_reward=+344.9 avg_steps=101 epsilon=0.050 avg_loss=4.5 time=0m21s total=31m02s +[ep_13000] ep=12901-13000 avg_reward=+292.4 avg_steps=83 epsilon=0.050 avg_loss=4.5 time=0m17s total=31m20s +[ep_13100] ep=13001-13100 avg_reward=+323.1 avg_steps=89 epsilon=0.050 avg_loss=4.6 time=0m19s total=31m39s +[ep_13200] ep=13101-13200 avg_reward=+310.8 avg_steps=86 epsilon=0.050 avg_loss=4.4 time=0m18s total=31m58s +[ep_13300] ep=13201-13300 avg_reward=+308.8 avg_steps=85 epsilon=0.050 avg_loss=4.4 time=0m18s total=32m16s +[ep_13400] ep=13301-13400 avg_reward=+395.9 avg_steps=111 epsilon=0.050 avg_loss=4.4 time=0m23s total=32m40s +[ep_13500] ep=13401-13500 avg_reward=+395.5 avg_steps=111 epsilon=0.050 avg_loss=4.3 time=0m23s total=33m03s +[ep_13600] ep=13501-13600 avg_reward=+384.9 avg_steps=107 epsilon=0.050 avg_loss=4.2 time=0m23s total=33m26s +[ep_13700] ep=13601-13700 avg_reward=+342.1 avg_steps=96 epsilon=0.050 avg_loss=4.3 time=0m20s total=33m47s +[ep_13800] ep=13701-13800 avg_reward=+319.3 avg_steps=92 epsilon=0.050 avg_loss=4.3 time=0m21s total=34m09s +[ep_13900] ep=13801-13900 avg_reward=+337.6 avg_steps=98 epsilon=0.050 avg_loss=4.2 time=0m25s total=34m35s +[ep_14000] ep=13901-14000 avg_reward=+302.8 avg_steps=85 epsilon=0.050 avg_loss=4.3 time=0m18s total=34m53s +[ep_14100] ep=14001-14100 avg_reward=+337.7 avg_steps=94 epsilon=0.050 avg_loss=4.4 time=0m20s total=35m13s +[ep_14200] ep=14101-14200 avg_reward=+315.7 avg_steps=88 epsilon=0.050 avg_loss=4.2 time=0m18s total=35m32s +[ep_14300] ep=14201-14300 avg_reward=+337.8 avg_steps=99 epsilon=0.050 avg_loss=4.3 time=0m21s total=35m53s +[ep_14400] ep=14301-14400 avg_reward=+313.3 avg_steps=88 epsilon=0.050 avg_loss=4.3 time=0m18s total=36m12s +[ep_14500] ep=14401-14500 avg_reward=+322.5 avg_steps=91 epsilon=0.050 avg_loss=4.3 time=0m19s total=36m32s +[ep_14600] ep=14501-14600 avg_reward=+385.1 avg_steps=106 epsilon=0.050 avg_loss=4.2 time=0m22s total=36m55s +[ep_14700] ep=14601-14700 avg_reward=+333.2 avg_steps=92 epsilon=0.050 avg_loss=4.1 time=0m19s total=37m14s +[ep_14800] ep=14701-14800 avg_reward=+325.7 avg_steps=92 epsilon=0.050 avg_loss=4.0 time=0m19s total=37m34s +[ep_14900] ep=14801-14900 avg_reward=+308.1 avg_steps=87 epsilon=0.050 avg_loss=4.2 time=0m18s total=37m53s +[ep_15000] ep=14901-15000 avg_reward=+374.5 avg_steps=103 epsilon=0.050 avg_loss=4.2 time=0m22s total=38m15s +[ep_15100] ep=15001-15100 avg_reward=+334.0 avg_steps=94 epsilon=0.050 avg_loss=4.3 time=0m20s total=38m35s +[ep_15200] ep=15101-15200 avg_reward=+363.1 avg_steps=97 epsilon=0.050 avg_loss=4.4 time=0m20s total=38m56s +[ep_15300] ep=15201-15300 avg_reward=+367.9 avg_steps=104 epsilon=0.050 avg_loss=4.5 time=0m22s total=39m18s +[ep_15400] ep=15301-15400 avg_reward=+367.8 avg_steps=99 epsilon=0.050 avg_loss=4.4 time=0m21s total=39m39s +[ep_15500] ep=15401-15500 avg_reward=+331.6 avg_steps=93 epsilon=0.050 avg_loss=4.2 time=0m19s total=39m59s +[ep_15600] ep=15501-15600 avg_reward=+349.4 avg_steps=97 epsilon=0.050 avg_loss=4.1 time=0m20s total=40m20s +[ep_15700] ep=15601-15700 avg_reward=+374.1 avg_steps=108 epsilon=0.050 avg_loss=4.1 time=0m23s total=40m43s +[ep_15800] ep=15701-15800 avg_reward=+345.4 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=41m04s +[ep_15900] ep=15801-15900 avg_reward=+372.3 avg_steps=108 epsilon=0.050 avg_loss=4.1 time=0m23s total=41m27s +[ep_16000] ep=15901-16000 avg_reward=+351.0 avg_steps=95 epsilon=0.050 avg_loss=4.2 time=0m20s total=41m47s +[ep_16100] ep=16001-16100 avg_reward=+345.8 avg_steps=99 epsilon=0.050 avg_loss=4.2 time=0m21s total=42m08s +[ep_16200] ep=16101-16200 avg_reward=+403.9 avg_steps=110 epsilon=0.050 avg_loss=4.2 time=0m23s total=42m32s +[ep_16300] ep=16201-16300 avg_reward=+424.1 avg_steps=116 epsilon=0.050 avg_loss=4.2 time=0m24s total=42m57s +[ep_16400] ep=16301-16400 avg_reward=+383.7 avg_steps=104 epsilon=0.050 avg_loss=4.3 time=0m22s total=43m19s +[ep_16500] ep=16401-16500 avg_reward=+356.4 avg_steps=97 epsilon=0.050 avg_loss=4.1 time=0m20s total=43m40s +[ep_16600] ep=16501-16600 avg_reward=+367.3 avg_steps=104 epsilon=0.050 avg_loss=4.1 time=0m22s total=44m02s +[ep_16700] ep=16601-16700 avg_reward=+322.2 avg_steps=96 epsilon=0.050 avg_loss=4.0 time=0m20s total=44m23s +[ep_16800] ep=16701-16800 avg_reward=+376.6 avg_steps=103 epsilon=0.050 avg_loss=4.1 time=0m22s total=44m45s +[ep_16900] ep=16801-16900 avg_reward=+340.6 avg_steps=96 epsilon=0.050 avg_loss=4.1 time=0m20s total=45m06s +[ep_17000] ep=16901-17000 avg_reward=+353.4 avg_steps=99 epsilon=0.050 avg_loss=4.0 time=0m21s total=45m27s +[ep_17100] ep=17001-17100 avg_reward=+304.6 avg_steps=87 epsilon=0.050 avg_loss=4.0 time=0m18s total=45m46s +[ep_17200] ep=17101-17200 avg_reward=+322.3 avg_steps=90 epsilon=0.050 avg_loss=3.9 time=0m19s total=46m05s +[ep_17300] ep=17201-17300 avg_reward=+376.1 avg_steps=105 epsilon=0.050 avg_loss=4.0 time=0m22s total=46m28s +[ep_17400] ep=17301-17400 avg_reward=+333.3 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=46m48s +[ep_17500] ep=17401-17500 avg_reward=+363.4 avg_steps=104 epsilon=0.050 avg_loss=4.0 time=0m22s total=47m10s +[ep_17600] ep=17501-17600 avg_reward=+336.3 avg_steps=93 epsilon=0.050 avg_loss=3.9 time=0m20s total=47m30s +[ep_17700] ep=17601-17700 avg_reward=+347.2 avg_steps=95 epsilon=0.050 avg_loss=4.1 time=0m20s total=47m51s +[ep_17800] ep=17701-17800 avg_reward=+344.4 avg_steps=98 epsilon=0.050 avg_loss=4.0 time=0m21s total=48m12s +[ep_17900] ep=17801-17900 avg_reward=+371.6 avg_steps=109 epsilon=0.050 avg_loss=4.0 time=0m23s total=48m35s +[ep_18000] ep=17901-18000 avg_reward=+433.1 avg_steps=119 epsilon=0.050 avg_loss=4.1 time=0m25s total=49m00s +[ep_18100] ep=18001-18100 avg_reward=+317.4 avg_steps=90 epsilon=0.050 avg_loss=4.2 time=0m19s total=49m20s +[ep_18200] ep=18101-18200 avg_reward=+381.8 avg_steps=106 epsilon=0.050 avg_loss=4.2 time=0m22s total=49m42s +[ep_18300] ep=18201-18300 avg_reward=+377.0 avg_steps=105 epsilon=0.050 avg_loss=4.1 time=0m22s total=50m05s +[ep_18400] ep=18301-18400 avg_reward=+377.0 avg_steps=101 epsilon=0.050 avg_loss=4.0 time=0m21s total=50m26s +[ep_18500] ep=18401-18500 avg_reward=+356.1 avg_steps=101 epsilon=0.050 avg_loss=4.0 time=0m21s total=50m48s +[ep_18600] ep=18501-18600 avg_reward=+409.6 avg_steps=109 epsilon=0.050 avg_loss=4.1 time=0m23s total=51m11s +[ep_18700] ep=18601-18700 avg_reward=+338.2 avg_steps=94 epsilon=0.050 avg_loss=4.1 time=0m20s total=51m31s +[ep_18800] ep=18701-18800 avg_reward=+354.6 avg_steps=104 epsilon=0.050 avg_loss=4.2 time=0m22s total=51m53s +[ep_18900] ep=18801-18900 avg_reward=+333.9 avg_steps=94 epsilon=0.050 avg_loss=4.2 time=0m20s total=52m14s +[ep_19000] ep=18901-19000 avg_reward=+388.2 avg_steps=106 epsilon=0.050 avg_loss=4.4 time=0m22s total=52m36s +[ep_19100] ep=19001-19100 avg_reward=+368.2 avg_steps=100 epsilon=0.050 avg_loss=4.3 time=0m21s total=52m58s +[ep_19200] ep=19101-19200 avg_reward=+435.8 avg_steps=121 epsilon=0.050 avg_loss=4.2 time=0m25s total=53m24s +[ep_19300] ep=19201-19300 avg_reward=+345.8 avg_steps=99 epsilon=0.050 avg_loss=4.1 time=0m21s total=53m45s +[ep_19400] ep=19301-19400 avg_reward=+372.5 avg_steps=104 epsilon=0.050 avg_loss=4.1 time=0m22s total=54m07s +[ep_19500] ep=19401-19500 avg_reward=+374.1 avg_steps=103 epsilon=0.050 avg_loss=3.9 time=0m21s total=54m29s +[ep_19600] ep=19501-19600 avg_reward=+387.1 avg_steps=109 epsilon=0.050 avg_loss=3.9 time=0m23s total=54m53s +[ep_19700] ep=19601-19700 avg_reward=+360.1 avg_steps=99 epsilon=0.050 avg_loss=3.9 time=0m21s total=55m14s +[ep_19800] ep=19701-19800 avg_reward=+387.3 avg_steps=107 epsilon=0.050 avg_loss=4.0 time=0m22s total=55m37s +[ep_19900] ep=19801-19900 avg_reward=+363.4 avg_steps=98 epsilon=0.050 avg_loss=3.9 time=0m20s total=55m58s +[ep_20000] ep=19901-20000 avg_reward=+335.6 avg_steps=94 epsilon=0.050 avg_loss=4.0 time=0m19s total=56m17s diff --git a/snake_observation.py b/snake_observation.py new file mode 100644 index 0000000..f858e87 --- /dev/null +++ b/snake_observation.py @@ -0,0 +1,31 @@ +"""A custom observation_function for retro.examples.snake. + +Crops the board to a window centered on the snake's head before encoding, +using egocentric_board()/encode_board() from retro_gamer.observation as +building blocks. This replaces retro_gamer's old built-in egocentric flags — +cropping is now just something an observation_function does for itself. + +Point a run's config.toml at this with: + + [metadata] + observation_function = "snake_observation:egocentric_observation" + board_size = [17, 17] # must match RADIUS below: 2*8+1 = 17 +""" +import numpy as np +from retro.views.headless import HeadlessView +from retro_gamer.observation import egocentric_board, encode_board, encode_state + +CHARACTER_SET = ["@", "*", ">", "<", "^", "v"] +RADIUS = 8 + + +def egocentric_observation(game): + """Board cropped to a (2*RADIUS+1)^2 window around the snake's head, plus apple_dx/apple_dy.""" + view = HeadlessView() + view.on_game_start(game) + view.render(game) + head = game.get_agent_by_name("Snake head") + cropped = egocentric_board(view.board_characters, head.position, RADIUS) + board_vec = encode_board(cropped, CHARACTER_SET).flatten() + extras = encode_state(game.state, ["apple_dx", "apple_dy"]) + return np.concatenate([board_vec, extras]) diff --git a/snake_training.md b/snake_training.md index c854dd1..5e57ba4 100644 --- a/snake_training.md +++ b/snake_training.md @@ -74,17 +74,18 @@ Huber loss (which is linear for large errors) break this cycle?** ## 5. Interpreting the training curve -Look at the snake training log. The reward climbs, then dips, then climbs again: +Look at `runs/snake/training.log`. The reward climbs, then dips, then climbs +again: ``` -[ep_1100] avg_reward=+34.5 avg_steps=57 -[ep_1800] avg_reward=+4.4 avg_steps=98 -[ep_3800] avg_reward=+51.2 avg_steps=33 -[ep_9000] avg_reward=+246.0 avg_steps=85 +[ep_1300] avg_reward= +31.6 avg_steps=49 +[ep_2300] avg_reward= -3.8 avg_steps=87 +[ep_4000] avg_reward= +53.2 avg_steps=42 +[ep_20000] avg_reward=+335.6 avg_steps=94 ``` -Notice that around episode 3,800, avg_steps dropped sharply (from ~98 to 33) -at the same time reward jumped. Then by episode 9,000, steps rose again while +Notice that around episode 4,000, avg_steps dropped sharply (from ~87 to 42) +at the same time reward jumped. Then by episode 20,000, steps rose again while reward kept climbing. **What do you think the agent was doing at each of these stages? Use the @@ -99,20 +100,20 @@ avg_steps and avg_reward numbers to support your interpretation.** Run these commands to watch the agent at three checkpoints: ``` -retro-gamer play runs/snake --checkpoint ep_1100 -retro-gamer play runs/snake --checkpoint ep_5400 -retro-gamer play runs/snake --checkpoint ep_17100 +retro-gamer play runs/snake --checkpoint ep_1300 +retro-gamer play runs/snake --checkpoint ep_4000 +retro-gamer play runs/snake --checkpoint ep_20000 ``` **Describe the agent's behavior at each checkpoint. What has the agent learned -by episode 5,400 that it hadn't yet learned at episode 1,100? What does the -episode 17,100 agent do that the earlier agents do not?** +by episode 4,000 that it hadn't yet learned at episode 1,300? What does the +episode 20,000 agent do that the earlier agents do not?** -*ep_1100:* +*ep_1300:* -*ep_5400:* +*ep_4000:* -*ep_17100:* +*ep_20000:* ---