Full run record (JSON)
{
"id": "20261022T060918_gemma-4-12b--unsloth--ud-q2_k_xl",
"quant_id": "gemma-4-12b--unsloth--ud-q2_k_xl",
"started_at": "2026-10-22T06:09:18Z",
"finished_at": "2026-10-22T06:17:48Z",
"status": "partial",
"machine": {
"name": "Ghost",
"gpu": "NVIDIA GeForce RTX 5070",
"vram_gb": 12,
"ram_gb": 64,
"cpu": "AMD Ryzen 7 5800X",
"os": "Windows 11",
"driver": "616.92",
"cuda": "13.4"
},
"software": {
"llama_cpp_build": "b11770 (39da130fc)",
"lm_eval_version": "0.4.13",
"harness_version": "0.1.2",
"python": "3.12.10"
},
"placement": "full_gpu",
"settings": {
"commands": [
"C:\\quantdiff\\llama.cpp\\bin\\llama-bench.exe -m C:\\quantdiff\\dl\\campaign\\aa6e1ccf5cd1\\gemma-4-12b-it-UD-Q2_K_XL.gguf -ngl 999 -fa 1 -p 512 -n 128 -r 5 -t 8 -o json",
"C:\\quantdiff\\llama.cpp\\bin\\llama-server.exe -m C:\\quantdiff\\dl\\campaign\\aa6e1ccf5cd1\\gemma-4-12b-it-UD-Q2_K_XL.gguf --host 127.0.0.1 --port 8089 -c 4096 -np 1 -ngl 999 -fa on --jinja --seed 1234 --no-webui -a quant"
],
"ctx": 4096,
"batch": 512,
"threads": 8,
"flash_attn": true,
"cache_type_k": "f16",
"cache_type_v": "f16",
"seed": 1234,
"temperature": 0.0,
"ngl": 999,
"gpu_info": "NVIDIA GeForce RTX 5070",
"server_ctx": 4096,
"server_ngl": 999,
"toolcall_set": "quantdiff-bfcl-v1",
"toolcall_set_sha256": "4d0e1667163c96a109965c58242673528c62318322450d9b218a730c09965020",
"toolcall_chat_template_kwargs": {
"enable_thinking": false
},
"toolcall_parallel_tool_calls": true,
"tasks": [
"gsm8k_cot_llama (150 items, quantdiff-lmeval-v1)",
"mmlu_llama (200 items, quantdiff-lmeval-v1)",
"arc_challenge_chat (150 items, quantdiff-lmeval-v1)"
]
},
"results": {
"speed": {
"pp_tokens": 512,
"tg_tokens": 128,
"pp_tps": 2508.86,
"pp_stddev": 71.35,
"tg_tps": 98.37,
"tg_stddev": 1.77
},
"toolcall": {
"score": 58.5,
"passed": 117,
"total": 200,
"request_errors": 0,
"by_category": {
"simple_python": {
"passed": 44,
"total": 60,
"score": 73.3
},
"multiple": {
"passed": 32,
"total": 40,
"score": 80.0
},
"parallel": {
"passed": 4,
"total": 30,
"score": 13.3
},
"parallel_multiple": {
"passed": 7,
"total": 30,
"score": 23.3
},
"irrelevance": {
"passed": 30,
"total": 40,
"score": 75.0
}
}
},
"tasks": {
"gsm8k": {
"metric": "exact_match,strict-match",
"value": 71.33,
"stderr": 3.7,
"n": 150,
"correct": 107,
"flexible_extract": 71.33,
"task": "gsm8k_cot_llama"
},
"mmlu": {
"metric": "letter_match (quantdiff letter-v2 (emphasis-tolerant; start, else last 'The best answer is X'), logged samples)",
"value": 63.0,
"stderr": 3.42,
"n": 200,
"correct": 126,
"task": "mmlu_llama",
"scorer": "letter-v2 (emphasis-tolerant; start, else last 'The best answer is X')",
"rescored": {
"date": "2026-10-22",
"correct_before": 126,
"rule_before": "letter-v1 (strict start)"
}
},
"arc_challenge": {
"metric": "letter_match (quantdiff letter-v2 (emphasis-tolerant; start, else last 'The best answer is X'), logged samples)",
"value": 68.67,
"stderr": 3.8,
"n": 150,
"correct": 103,
"task": "arc_challenge_chat",
"scorer": "letter-v2 (emphasis-tolerant; start, else last 'The best answer is X')",
"rescored": {
"date": "2026-10-22",
"correct_before": 95,
"rule_before": "letter-v1 (strict start)"
}
}
}
},
"logs": {
"llama_bench": "llama_bench.log",
"llama_server": "llama_server.log",
"toolcall": "toolcall.jsonl",
"lm_eval_summary": "lm_eval/tasks.json",
"lm_eval_results": "lm_eval/lm_eval_results.json",
"lm_eval_samples (raw answers, gz)": "lm_eval/samples.jsonl.gz",
"lm_eval_llama_server": "lm_eval/llama_server.log"
},
"example": false,
"notes": "",
"tasks_run": {
"suite": "quantdiff-lmeval-v1",
"subset_sha256": "b447babaa655053a38576dd86ac6eec83d1d75697655b230a99b54065c86fea2",
"machine": {
"name": "Ghost",
"hardware": "NVIDIA GeForce RTX 5070 12 GB (64 GB RAM)",
"backend": "CUDA",
"os": "Windows 11",
"placement": "full_gpu",
"ngl": 999
},
"software": {
"lm_eval_version": "0.4.13",
"python": "3.12.10",
"llama_cpp_build": "version: 0.7.0-dev (build 11770, commit 39da130fc)"
},
"settings": {
"suite": "quantdiff-lmeval-v1",
"lm_eval_model": "local-chat-completions",
"apply_chat_template": true,
"fewshot_as_multiturn": true,
"temperature": 0,
"max_gen_toks": 1024,
"seed": 1234,
"num_concurrent": 4,
"server_ctx": 16384,
"server_parallel": 4,
"flash_attn": true,
"server_seed": 1234,
"chat_template_kwargs": {
"enable_thinking": false
}
},
"commands": [
"C:\\quantdiff\\llama.cpp\\bin\\llama-server.exe -m C:\\quantdiff\\dl\\campaign\\aa6e1ccf5cd1\\gemma-4-12b-it-UD-Q2_K_XL.gguf --host 127.0.0.1 --port 8091 -c 16384 -np 4 -ngl 999 -fa on --jinja --seed 1234 --chat-template-kwargs \"{\\\"enable_thinking\\\": false}\" --no-webui -a quant",
"C:\\quantdiff\\venv\\Scripts\\python.exe -m lm_eval run --model local-chat-completions --model_args model=quant,base_url=http://127.0.0.1:8094/v1/chat/completions,num_concurrent=4,max_retries=3,tokenized_requests=False,timeout=1800 --tasks arc_challenge_chat,gsm8k_cot_llama,mmlu_llama_abstract_algebra,mmlu_llama_anatomy,mmlu_llama_astronomy,mmlu_llama_business_ethics,mmlu_llama_clinical_knowledge,mmlu_llama_college_biology,mmlu_llama_college_chemistry,mmlu_llama_college_mathematics,mmlu_llama_college_medicine,mmlu_llama_college_physics,mmlu_llama_conceptual_physics,mmlu_llama_electrical_engineering,mmlu_llama_elementary_mathematics,mmlu_llama_global_facts,mmlu_llama_high_school_biology,mmlu_llama_high_school_chemistry,mmlu_llama_high_school_computer_science,mmlu_llama_high_school_geography,mmlu_llama_high_school_government_and_politics,mmlu_llama_high_school_macroeconomics,mmlu_llama_high_school_mathematics,mmlu_llama_high_school_microeconomics,mmlu_llama_high_school_physics,mmlu_llama_high_school_psychology,mmlu_llama_high_school_statistics,mmlu_llama_high_school_us_history,mmlu_llama_high_school_world_history,mmlu_llama_human_aging,mmlu_llama_human_sexuality,mmlu_llama_international_law,mmlu_llama_jurisprudence,mmlu_llama_logical_fallacies,mmlu_llama_management,mmlu_llama_marketing,mmlu_llama_medical_genetics,mmlu_llama_miscellaneous,mmlu_llama_moral_disputes,mmlu_llama_moral_scenarios,mmlu_llama_nutrition,mmlu_llama_philosophy,mmlu_llama_prehistory,mmlu_llama_professional_accounting,mmlu_llama_professional_law,mmlu_llama_professional_medicine,mmlu_llama_professional_psychology,mmlu_llama_public_relations,mmlu_llama_security_studies,mmlu_llama_sociology,mmlu_llama_us_foreign_policy,mmlu_llama_virology,mmlu_llama_world_religions --apply_chat_template --fewshot_as_multiturn true --samples {\\\"arc_challenge_chat\\\":[11,15,29,32,35,39,55,63,71,77,81,82,84,92,94,97,98,110,123,129,132,135,140,142,144,149,154,156,160,171,179,182,185,186,201,207,211,221,227,234,239,244,253,268,284,289,294,295,296,304,313,316,317,321,335,338,342,348,350,357,363,374,380,392,398,409,413,416,420,438,440,449,483,484,510,511,523,527,545,551,552,556,563,566,586,605,614,618,619,625,649,662,666,683,699,707,708,709,715,721,726,727,739,771,772,790,809,812,852,862,873,893,896,900,902,914,928,930,941,943,949,951,954,958,964,967,968,974,985,991,993,996,1015,1016,1023,1025,1030,1037,1040,1054,1063,1084,1085,1094,1108,1118,1124,1139,1141,1153],\\\"gsm8k_cot_llama\\\":[3,5,37,55,57,58,76,81,90,107,124,133,139,141,148,150,169,173,179,190,212,232,244,264,280,286,298,303,310,317,321,332,336,338,346,380,398,417,422,424,427,438,444,449,470,476,498,502,526,533,535,547,561,566,569,571,573,576,593,601,609,611,623,624,630,641,653,657,664,668,677,678,679,684,695,709,712,714,715,727,750,756,761,768,779,780,801,805,808,815,819,820,840,864,878,886,889,902,914,936,938,958,971,974,978,982,983,994,1004,1022,1032,1034,1037,1042,1045,1055,1056,1062,1076,1079,1088,1090,1093,1095,1100,1122,1150,1152,1155,1157,1159,1163,1183,1187,1193,1194,1198,1203,1204,1206,1211,1227,1229,1233,1249,1269,1275,1286,1294,1307],\\\"mmlu_llama_abstract_algebra\\\":[52],\\\"mmlu_llama_anatomy\\\":[43],\\\"mmlu_llama_astronomy\\\":[56],\\\"mmlu_llama_business_ethics\\\":[94],\\\"mmlu_llama_clinical_knowledge\\\":[1,93,182,244,254],\\\"mmlu_llama_college_biology\\\":[59,107],\\\"mmlu_llama_college_chemistry\\\":[16],\\\"mmlu_llama_college_mathematics\\\":[29,65],\\\"mmlu_llama_college_medicine\\\":[27,50,117],\\\"mmlu_llama_college_physics\\\":[37],\\\"mmlu_llama_conceptual_physics\\\":[14,63,109,167,196],\\\"mmlu_llama_electrical_engineering\\\":[53],\\\"mmlu_llama_elementary_mathematics\\\":[16,95,105,252,358],\\\"mmlu_llama_global_facts\\\":[22],\\\"mmlu_llama_high_school_biology\\\":[103,159,182,185,211,219],\\\"mmlu_llama_high_school_chemistry\\\":[34,106,162],\\\"mmlu_llama_high_school_computer_science\\\":[36,53,65,88],\\\"mmlu_llama_high_school_geography\\\":[30,133],\\\"mmlu_llama_high_school_government_and_politics\\\":[89,191],\\\"mmlu_llama_high_school_macroeconomics\\\":[77,78,127,335,354],\\\"mmlu_llama_high_school_mathematics\\\":[106,130,220,243,251],\\\"mmlu_llama_high_school_microeconomics\\\":[175,227],\\\"mmlu_llama_high_school_physics\\\":[63,67,74],\\\"mmlu_llama_high_school_psychology\\\":[237,260,290,348,392],\\\"mmlu_llama_high_school_statistics\\\":[61,71,143],\\\"mmlu_llama_high_school_us_history\\\":[23,149,167,194],\\\"mmlu_llama_high_school_world_history\\\":[71,93,175,195,200,233],\\\"mmlu_llama_human_aging\\\":[0,2,22,100,133],\\\"mmlu_llama_human_sexuality\\\":[113],\\\"mmlu_llama_international_law\\\":[55],\\\"mmlu_llama_jurisprudence\\\":[14,79],\\\"mmlu_llama_logical_fallacies\\\":[15,19,22,35,127],\\\"mmlu_llama_management\\\":[39],\\\"mmlu_llama_marketing\\\":[184,220],\\\"mmlu_llama_medical_genetics\\\":[11,57],\\\"mmlu_llama_miscellaneous\\\":[101,205,245,284,305,364,483,498,581,613,689],\\\"mmlu_llama_moral_disputes\\\":[55,259],\\\"mmlu_llama_moral_scenarios\\\":[27,42,86,155,172,235,288,320,392,403,438,458,466,624,843],\\\"mmlu_llama_nutrition\\\":[95,144,198,212,255,282],\\\"mmlu_llama_philosophy\\\":[10,79,96,114,151,182,200,206,242],\\\"mmlu_llama_prehistory\\\":[1,104,139,186],\\\"mmlu_llama_professional_accounting\\\":[127,180],\\\"mmlu_llama_professional_law\\\":[31,39,210,416,424,428,440,442,459,658,724,753,754,766,813,1026,1036,1059,1234,1279,1313,1336,1339,1428,1454,1522],\\\"mmlu_llama_professional_medicine\\\":[11,201,206],\\\"mmlu_llama_professional_psychology\\\":[62,164,172,266,316,414,579],\\\"mmlu_llama_public_relations\\\":[107],\\\"mmlu_llama_security_studies\\\":[5,46,74,152,169],\\\"mmlu_llama_sociology\\\":[163,167],\\\"mmlu_llama_us_foreign_policy\\\":[58,60,74],\\\"mmlu_llama_virology\\\":[29,66],\\\"mmlu_llama_world_religions\\\":[7,59,118]} --gen_kwargs temperature=0,max_gen_toks=1024 --seed 1234 --use_cache C:\\quantdiff\\results\\tasks\\20261022T060918_gemma-4-12b--unsloth--ud-q2_k_xl\\cache\\responses --output_path C:\\quantdiff\\results\\tasks\\20261022T060918_gemma-4-12b--unsloth--ud-q2_k_xl\\lm_eval --log_samples"
],
"started_at": "2026-10-22T03:09:17-04:00",
"finished_at": "2026-10-22T03:18:45-04:00",
"seconds": {
"download": 0,
"eval": 547,
"total": 568
},
"quant": {
"repo": "unsloth/gemma-4-12b-it-GGUF",
"file": "gemma-4-12b-it-UD-Q2_K_XL.gguf",
"revision": "fc034cfff751157913579611efad8462ac1be606",
"sha256": "aa6e1ccf5cd1c5340c4786f0bda2478181327f14aba2359e51e455d02c90bff3"
}
}
}