pawn-base / eval_results.json
thomas-schweich's picture
Update eval_results.json: probes + diagnostics from run_evals_backbone.py
c62ea6a verified
Raw
History Blame Contribute Delete
20.2 kB
{
"probes": {
"piece_type": {
"embed": {
"accuracy": 0.7939537763595581,
"loss": 0.8468984961509705,
"best_accuracy": 0.7941248416900635,
"n_train": 721682,
"n_val": 175570
},
"layer_0": {
"accuracy": 0.8614519834518433,
"loss": 0.48088300228118896,
"best_accuracy": 0.8614519834518433,
"n_train": 721682,
"n_val": 175570
},
"layer_1": {
"accuracy": 0.899398922920227,
"loss": 0.3235924243927002,
"best_accuracy": 0.899398922920227,
"n_train": 721682,
"n_val": 175570
},
"layer_2": {
"accuracy": 0.8952611088752747,
"loss": 0.37268662452697754,
"best_accuracy": 0.8952611088752747,
"n_train": 721682,
"n_val": 175570
},
"layer_3": {
"accuracy": 0.91811603307724,
"loss": 0.28765103220939636,
"best_accuracy": 0.91811603307724,
"n_train": 721682,
"n_val": 175570
},
"layer_4": {
"accuracy": 0.9070653319358826,
"loss": 0.3429071307182312,
"best_accuracy": 0.9071179628372192,
"n_train": 721682,
"n_val": 175570
},
"layer_5": {
"accuracy": 0.8869622945785522,
"loss": 0.46307042241096497,
"best_accuracy": 0.8869622945785522,
"n_train": 721682,
"n_val": 175570
},
"layer_6": {
"accuracy": 0.8946530818939209,
"loss": 0.6553605198860168,
"best_accuracy": 0.8946530818939209,
"n_train": 721682,
"n_val": 175570
},
"layer_7": {
"accuracy": 0.8841903805732727,
"loss": 0.8387924432754517,
"best_accuracy": 0.8852172493934631,
"n_train": 721682,
"n_val": 175570
}
},
"side_to_move": {
"embed": {
"accuracy": 0.6726946234703064,
"loss": 0.617131233215332,
"best_accuracy": 0.6737483739852905,
"n_train": 721682,
"n_val": 175570
},
"layer_0": {
"accuracy": 0.7186421155929565,
"loss": 0.5007065534591675,
"best_accuracy": 0.7200603485107422,
"n_train": 721682,
"n_val": 175570
},
"layer_1": {
"accuracy": 0.9996126890182495,
"loss": 0.0013729166239500046,
"best_accuracy": 0.9997152090072632,
"n_train": 721682,
"n_val": 175570
},
"layer_2": {
"accuracy": 1.0,
"loss": 1.0403724104435241e-07,
"best_accuracy": 1.0,
"n_train": 721682,
"n_val": 175570
},
"layer_3": {
"accuracy": 1.0,
"loss": 2.8485755976248583e-09,
"best_accuracy": 1.0,
"n_train": 721682,
"n_val": 175570
},
"layer_4": {
"accuracy": 1.0,
"loss": 2.5146256188435245e-09,
"best_accuracy": 1.0,
"n_train": 721682,
"n_val": 175570
},
"layer_5": {
"accuracy": 1.0,
"loss": 1.8359436193549072e-09,
"best_accuracy": 1.0,
"n_train": 721682,
"n_val": 175570
},
"layer_6": {
"accuracy": 1.0,
"loss": 5.28788575593353e-07,
"best_accuracy": 1.0,
"n_train": 721682,
"n_val": 175570
},
"layer_7": {
"accuracy": 0.9997778534889221,
"loss": 0.002000772627070546,
"best_accuracy": 0.9998462200164795,
"n_train": 721682,
"n_val": 175570
}
},
"is_check": {
"embed": {
"accuracy": 0.9438571333885193,
"loss": 0.2128543257713318,
"best_accuracy": 0.9438571333885193,
"n_train": 721682,
"n_val": 175570
},
"layer_0": {
"accuracy": 0.9444608688354492,
"loss": 0.1777006834745407,
"best_accuracy": 0.9445463418960571,
"n_train": 721682,
"n_val": 175570
},
"layer_1": {
"accuracy": 0.9497294425964355,
"loss": 0.13770730793476105,
"best_accuracy": 0.9497294425964355,
"n_train": 721682,
"n_val": 175570
},
"layer_2": {
"accuracy": 0.9498946070671082,
"loss": 0.1334017962217331,
"best_accuracy": 0.9500312805175781,
"n_train": 721682,
"n_val": 175570
},
"layer_3": {
"accuracy": 0.948766827583313,
"loss": 0.13876499235630035,
"best_accuracy": 0.9489434361457825,
"n_train": 721682,
"n_val": 175570
},
"layer_4": {
"accuracy": 0.948692798614502,
"loss": 0.13251903653144836,
"best_accuracy": 0.9488409161567688,
"n_train": 721682,
"n_val": 175570
},
"layer_5": {
"accuracy": 0.9457025527954102,
"loss": 0.15899775922298431,
"best_accuracy": 0.9465056657791138,
"n_train": 721682,
"n_val": 175570
},
"layer_6": {
"accuracy": 0.9412370920181274,
"loss": 0.22216182947158813,
"best_accuracy": 0.9436520934104919,
"n_train": 721682,
"n_val": 175570
},
"layer_7": {
"accuracy": 0.9402061700820923,
"loss": 0.24624347686767578,
"best_accuracy": 0.9422793984413147,
"n_train": 721682,
"n_val": 175570
}
},
"castling_rights": {
"embed": {
"accuracy": 0.9315600395202637,
"loss": 0.22545865178108215,
"best_accuracy": 0.9315600395202637,
"n_train": 721682,
"n_val": 175570
},
"layer_0": {
"accuracy": 0.9853334426879883,
"loss": 0.03903070092201233,
"best_accuracy": 0.9853420257568359,
"n_train": 721682,
"n_val": 175570
},
"layer_1": {
"accuracy": 0.9914606213569641,
"loss": 0.021991031244397163,
"best_accuracy": 0.991588830947876,
"n_train": 721682,
"n_val": 175570
},
"layer_2": {
"accuracy": 0.9929686188697815,
"loss": 0.017853515222668648,
"best_accuracy": 0.9930654168128967,
"n_train": 721682,
"n_val": 175570
},
"layer_3": {
"accuracy": 0.9865978956222534,
"loss": 0.03318539634346962,
"best_accuracy": 0.9868285655975342,
"n_train": 721682,
"n_val": 175570
},
"layer_4": {
"accuracy": 0.9831904172897339,
"loss": 0.04187026247382164,
"best_accuracy": 0.9836090803146362,
"n_train": 721682,
"n_val": 175570
},
"layer_5": {
"accuracy": 0.9788687825202942,
"loss": 0.058379411697387695,
"best_accuracy": 0.9798797965049744,
"n_train": 721682,
"n_val": 175570
},
"layer_6": {
"accuracy": 0.9751836657524109,
"loss": 0.0907355323433876,
"best_accuracy": 0.9754883646965027,
"n_train": 721682,
"n_val": 175570
},
"layer_7": {
"accuracy": 0.9707794189453125,
"loss": 0.11967474222183228,
"best_accuracy": 0.9712464809417725,
"n_train": 721682,
"n_val": 175570
}
},
"ep_square": {
"embed": {
"accuracy": 0.9986899495124817,
"loss": 0.01372161041945219,
"best_accuracy": 0.9986899495124817,
"n_train": 721682,
"n_val": 175570
},
"layer_0": {
"accuracy": 0.998684287071228,
"loss": 0.011701500043272972,
"best_accuracy": 0.9986899495124817,
"n_train": 721682,
"n_val": 175570
},
"layer_1": {
"accuracy": 0.9981716275215149,
"loss": 0.006189861800521612,
"best_accuracy": 0.9985817670822144,
"n_train": 721682,
"n_val": 175570
},
"layer_2": {
"accuracy": 0.9982627630233765,
"loss": 0.006863097194582224,
"best_accuracy": 0.9985817670822144,
"n_train": 721682,
"n_val": 175570
},
"layer_3": {
"accuracy": 0.9990431070327759,
"loss": 0.003790292190387845,
"best_accuracy": 0.9991456270217896,
"n_train": 721682,
"n_val": 175570
},
"layer_4": {
"accuracy": 0.9987013339996338,
"loss": 0.007270838599652052,
"best_accuracy": 0.9988323450088501,
"n_train": 721682,
"n_val": 175570
},
"layer_5": {
"accuracy": 0.9974995255470276,
"loss": 0.026773931458592415,
"best_accuracy": 0.9982684850692749,
"n_train": 721682,
"n_val": 175570
},
"layer_6": {
"accuracy": 0.9960300326347351,
"loss": 0.0977015346288681,
"best_accuracy": 0.9980349540710449,
"n_train": 721682,
"n_val": 175570
},
"layer_7": {
"accuracy": 0.9952667951583862,
"loss": 0.13185012340545654,
"best_accuracy": 0.9978071451187134,
"n_train": 721682,
"n_val": 175570
}
},
"material_count": {
"embed": {
"accuracy": 0.034823477268218994,
"loss": 1.8249613046646118,
"best_accuracy": 0.03692054748535156,
"n_train": 721682,
"n_val": 175570,
"mae": 9.569934844970703
},
"layer_0": {
"accuracy": 0.7515236735343933,
"loss": 0.46982043981552124,
"best_accuracy": 0.7537726759910583,
"n_train": 721682,
"n_val": 175570,
"mae": 5.040376663208008
},
"layer_1": {
"accuracy": 0.7843592762947083,
"loss": 0.4077347218990326,
"best_accuracy": 0.8040273189544678,
"n_train": 721682,
"n_val": 175570,
"mae": 4.520277500152588
},
"layer_2": {
"accuracy": 0.7963148355484009,
"loss": 0.3851289749145508,
"best_accuracy": 0.8129063844680786,
"n_train": 721682,
"n_val": 175570,
"mae": 4.381388187408447
},
"layer_3": {
"accuracy": 0.8059321641921997,
"loss": 0.3669446110725403,
"best_accuracy": 0.8202922940254211,
"n_train": 721682,
"n_val": 175570,
"mae": 4.309057712554932
},
"layer_4": {
"accuracy": 0.8212227821350098,
"loss": 0.33803296089172363,
"best_accuracy": 0.8356728553771973,
"n_train": 721682,
"n_val": 175570,
"mae": 4.076085090637207
},
"layer_5": {
"accuracy": 0.833043098449707,
"loss": 0.3156830370426178,
"best_accuracy": 0.8350316882133484,
"n_train": 721682,
"n_val": 175570,
"mae": 3.93052077293396
},
"layer_6": {
"accuracy": 0.8065394163131714,
"loss": 0.36579638719558716,
"best_accuracy": 0.81328946352005,
"n_train": 721682,
"n_val": 175570,
"mae": 4.352488994598389
},
"layer_7": {
"accuracy": 0.7795088887214661,
"loss": 0.41690582036972046,
"best_accuracy": 0.7882944345474243,
"n_train": 721682,
"n_val": 175570,
"mae": 4.678890228271484
}
},
"legal_move_count": {
"embed": {
"accuracy": 0.008694469928741455,
"loss": 149.88076782226562,
"best_accuracy": 0.00975710153579712,
"n_train": 721682,
"n_val": 175570,
"mae": 10.171092987060547
},
"layer_0": {
"accuracy": 0.43983858823776245,
"loss": 84.69378662109375,
"best_accuracy": 0.43983858823776245,
"n_train": 721682,
"n_val": 175570,
"mae": 7.150360584259033
},
"layer_1": {
"accuracy": 0.5228815078735352,
"loss": 72.13809204101562,
"best_accuracy": 0.5228815078735352,
"n_train": 721682,
"n_val": 175570,
"mae": 6.5377326011657715
},
"layer_2": {
"accuracy": 0.5866934061050415,
"loss": 62.49002456665039,
"best_accuracy": 0.5866934061050415,
"n_train": 721682,
"n_val": 175570,
"mae": 6.011066436767578
},
"layer_3": {
"accuracy": 0.5604725480079651,
"loss": 66.45449829101562,
"best_accuracy": 0.5607300996780396,
"n_train": 721682,
"n_val": 175570,
"mae": 6.218656063079834
},
"layer_4": {
"accuracy": 0.6754791140556335,
"loss": 49.0660400390625,
"best_accuracy": 0.6755585670471191,
"n_train": 721682,
"n_val": 175570,
"mae": 5.165564060211182
},
"layer_5": {
"accuracy": 0.6420835256576538,
"loss": 54.11530303955078,
"best_accuracy": 0.6492248773574829,
"n_train": 721682,
"n_val": 175570,
"mae": 5.334083557128906
},
"layer_6": {
"accuracy": 0.5348567962646484,
"loss": 70.32747650146484,
"best_accuracy": 0.567049503326416,
"n_train": 721682,
"n_val": 175570,
"mae": 6.1242218017578125
},
"layer_7": {
"accuracy": 0.39110326766967773,
"loss": 92.06233978271484,
"best_accuracy": 0.42400068044662476,
"n_train": 721682,
"n_val": 175570,
"mae": 7.249655723571777
}
},
"halfmove_clock": {
"embed": {
"accuracy": 0.005232453346252441,
"loss": 492.56304931640625,
"best_accuracy": 0.006348371505737305,
"n_train": 721682,
"n_val": 175570,
"mae": 14.510623931884766
},
"layer_0": {
"accuracy": 0.37127983570098877,
"loss": 311.3132629394531,
"best_accuracy": 0.37127983570098877,
"n_train": 721682,
"n_val": 175570,
"mae": 11.005152702331543
},
"layer_1": {
"accuracy": 0.43745899200439453,
"loss": 278.5444030761719,
"best_accuracy": 0.43745899200439453,
"n_train": 721682,
"n_val": 175570,
"mae": 10.431397438049316
},
"layer_2": {
"accuracy": 0.4526997208595276,
"loss": 270.9978942871094,
"best_accuracy": 0.4536823034286499,
"n_train": 721682,
"n_val": 175570,
"mae": 10.503276824951172
},
"layer_3": {
"accuracy": 0.4739035964012146,
"loss": 260.4986877441406,
"best_accuracy": 0.4743286967277527,
"n_train": 721682,
"n_val": 175570,
"mae": 10.437573432922363
},
"layer_4": {
"accuracy": 0.45564568042755127,
"loss": 269.5391540527344,
"best_accuracy": 0.45679765939712524,
"n_train": 721682,
"n_val": 175570,
"mae": 10.679705619812012
},
"layer_5": {
"accuracy": 0.4151163101196289,
"loss": 289.6074523925781,
"best_accuracy": 0.4151163101196289,
"n_train": 721682,
"n_val": 175570,
"mae": 11.022024154663086
},
"layer_6": {
"accuracy": 0.3814623951911926,
"loss": 306.27130126953125,
"best_accuracy": 0.3814623951911926,
"n_train": 721682,
"n_val": 175570,
"mae": 11.431619644165039
},
"layer_7": {
"accuracy": 0.3014959692955017,
"loss": 345.86700439453125,
"best_accuracy": 0.3069889545440674,
"n_train": 721682,
"n_val": 175570,
"mae": 12.022778511047363
}
},
"game_phase": {
"embed": {
"accuracy": 0.622520923614502,
"loss": 0.7939810752868652,
"best_accuracy": 0.6229594945907593,
"n_train": 721682,
"n_val": 175570
},
"layer_0": {
"accuracy": 0.9163581132888794,
"loss": 0.1934569925069809,
"best_accuracy": 0.9167283773422241,
"n_train": 721682,
"n_val": 175570
},
"layer_1": {
"accuracy": 0.9480605721473694,
"loss": 0.120170958340168,
"best_accuracy": 0.9480605721473694,
"n_train": 721682,
"n_val": 175570
},
"layer_2": {
"accuracy": 0.9529076218605042,
"loss": 0.10889476537704468,
"best_accuracy": 0.9529076218605042,
"n_train": 721682,
"n_val": 175570
},
"layer_3": {
"accuracy": 0.9525943994522095,
"loss": 0.11166463047266006,
"best_accuracy": 0.9525943994522095,
"n_train": 721682,
"n_val": 175570
},
"layer_4": {
"accuracy": 0.9572990536689758,
"loss": 0.10107019543647766,
"best_accuracy": 0.9584438800811768,
"n_train": 721682,
"n_val": 175570
},
"layer_5": {
"accuracy": 0.9517115354537964,
"loss": 0.13210734724998474,
"best_accuracy": 0.9521843194961548,
"n_train": 721682,
"n_val": 175570
},
"layer_6": {
"accuracy": 0.9353192448616028,
"loss": 0.27813100814819336,
"best_accuracy": 0.9375462532043457,
"n_train": 721682,
"n_val": 175570
},
"layer_7": {
"accuracy": 0.9186364412307739,
"loss": 0.38030770421028137,
"best_accuracy": 0.9187902212142944,
"n_train": 721682,
"n_val": 175570
}
}
},
"diagnostics": {
"in_check": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.9907619999999999,
"std_legal_rate": 0.030408211982949608,
"mean_pad_prob": 0.0007640491741631124,
"mean_entropy": 1.360506830084973,
"std_entropy": 0.4851868447382186
},
"double_check": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.968241,
"std_legal_rate": 0.06678679449561867,
"mean_pad_prob": 0.001569394861201583,
"mean_entropy": 1.098989733998195,
"std_entropy": 0.41889251754830115
},
"pin_restricts": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.9892730000000002,
"std_legal_rate": 0.021233027834013693,
"mean_pad_prob": 8.378310395977987e-05,
"mean_entropy": 3.179187242682418,
"std_entropy": 0.614420535115668
},
"ep_available": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.995693,
"std_legal_rate": 0.01136044677818615,
"mean_pad_prob": 4.877161065352252e-05,
"mean_entropy": 3.465905277723074,
"std_entropy": 0.3122574852442407
},
"castle_legal_k": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.9979520000000001,
"std_legal_rate": 0.0048400099173452156,
"mean_pad_prob": 7.714506284892467e-08,
"mean_entropy": 3.5851313720464706,
"std_entropy": 0.1854135106810934
},
"castle_legal_q": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.9969480000000002,
"std_legal_rate": 0.006473430002711087,
"mean_pad_prob": 8.240626183083216e-08,
"mean_entropy": 3.6590794766902923,
"std_entropy": 0.18080142984059425
},
"castle_blocked_check": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.9961120000000001,
"std_legal_rate": 0.010133284561286142,
"mean_pad_prob": 0.0004383320852224248,
"mean_entropy": 1.2391307091199792,
"std_entropy": 0.5261870073605922
},
"promotion_available": {
"n_positions": 10000,
"terminal": false,
"mean_legal_rate": 0.997017,
"std_legal_rate": 0.007707899259850253,
"mean_pad_prob": 9.902012749442277e-06,
"mean_entropy": 3.209924154102802,
"std_entropy": 0.479078440343986
},
"checkmate": {
"n_positions": 10000,
"terminal": true,
"mean_legal_rate": 0.0,
"std_legal_rate": 0.0,
"mean_pad_prob": 0.9151065862635662,
"mean_entropy": 0.314861747341415,
"std_entropy": 0.4386393359006536
},
"stalemate": {
"n_positions": 10000,
"terminal": true,
"mean_legal_rate": 0.0,
"std_legal_rate": 0.0,
"mean_pad_prob": 0.9670851592363986,
"mean_entropy": 0.09786651973089533,
"std_entropy": 0.2625593175802058
}
}
}