{
  "run_id": "20261022T060918_gemma-4-12b--unsloth--ud-q2_k_xl",
  "quant_id": "gemma-4-12b--unsloth--ud-q2_k_xl",
  "suite": "quantdiff-lmeval-v1",
  "subset_sha256": "b447babaa655053a38576dd86ac6eec83d1d75697655b230a99b54065c86fea2",
  "machine": {
    "name": "Ghost",
    "hardware": "NVIDIA GeForce RTX 5070 12 GB (64 GB RAM)",
    "backend": "CUDA",
    "os": "Windows 11",
    "placement": "full_gpu",
    "ngl": 999
  },
  "software": {
    "lm_eval_version": "0.4.13",
    "python": "3.12.10",
    "llama_cpp_build": "version: 0.7.0-dev (build 11770, commit 39da130fc)"
  },
  "settings": {
    "suite": "quantdiff-lmeval-v1",
    "lm_eval_model": "local-chat-completions",
    "apply_chat_template": true,
    "fewshot_as_multiturn": true,
    "temperature": 0,
    "max_gen_toks": 1024,
    "seed": 1234,
    "num_concurrent": 4,
    "server_ctx": 16384,
    "server_parallel": 4,
    "flash_attn": true,
    "server_seed": 1234,
    "chat_template_kwargs": {
      "enable_thinking": false
    }
  },
  "commands": [
    "C:\\quantdiff\\llama.cpp\\bin\\llama-server.exe -m C:\\quantdiff\\dl\\campaign\\aa6e1ccf5cd1\\gemma-4-12b-it-UD-Q2_K_XL.gguf --host 127.0.0.1 --port 8091 -c 16384 -np 4 -ngl 999 -fa on --jinja --seed 1234 --chat-template-kwargs \"{\\\"enable_thinking\\\": false}\" --no-webui -a quant",
    "C:\\quantdiff\\venv\\Scripts\\python.exe -m lm_eval run --model local-chat-completions --model_args model=quant,base_url=http://127.0.0.1:8094/v1/chat/completions,num_concurrent=4,max_retries=3,tokenized_requests=False,timeout=1800 --tasks arc_challenge_chat,gsm8k_cot_llama,mmlu_llama_abstract_algebra,mmlu_llama_anatomy,mmlu_llama_astronomy,mmlu_llama_business_ethics,mmlu_llama_clinical_knowledge,mmlu_llama_college_biology,mmlu_llama_college_chemistry,mmlu_llama_college_mathematics,mmlu_llama_college_medicine,mmlu_llama_college_physics,mmlu_llama_conceptual_physics,mmlu_llama_electrical_engineering,mmlu_llama_elementary_mathematics,mmlu_llama_global_facts,mmlu_llama_high_school_biology,mmlu_llama_high_school_chemistry,mmlu_llama_high_school_computer_science,mmlu_llama_high_school_geography,mmlu_llama_high_school_government_and_politics,mmlu_llama_high_school_macroeconomics,mmlu_llama_high_school_mathematics,mmlu_llama_high_school_microeconomics,mmlu_llama_high_school_physics,mmlu_llama_high_school_psychology,mmlu_llama_high_school_statistics,mmlu_llama_high_school_us_history,mmlu_llama_high_school_world_history,mmlu_llama_human_aging,mmlu_llama_human_sexuality,mmlu_llama_international_law,mmlu_llama_jurisprudence,mmlu_llama_logical_fallacies,mmlu_llama_management,mmlu_llama_marketing,mmlu_llama_medical_genetics,mmlu_llama_miscellaneous,mmlu_llama_moral_disputes,mmlu_llama_moral_scenarios,mmlu_llama_nutrition,mmlu_llama_philosophy,mmlu_llama_prehistory,mmlu_llama_professional_accounting,mmlu_llama_professional_law,mmlu_llama_professional_medicine,mmlu_llama_professional_psychology,mmlu_llama_public_relations,mmlu_llama_security_studies,mmlu_llama_sociology,mmlu_llama_us_foreign_policy,mmlu_llama_virology,mmlu_llama_world_religions --apply_chat_template --fewshot_as_multiturn true --samples {\\\"arc_challenge_chat\\\":[11,15,29,32,35,39,55,63,71,77,81,82,84,92,94,97,98,110,123,129,132,135,140,142,144,149,154,156,160,171,179,182,185,186,201,207,211,221,227,234,239,244,253,268,284,289,294,295,296,304,313,316,317,321,335,338,342,348,350,357,363,374,380,392,398,409,413,416,420,438,440,449,483,484,510,511,523,527,545,551,552,556,563,566,586,605,614,618,619,625,649,662,666,683,699,707,708,709,715,721,726,727,739,771,772,790,809,812,852,862,873,893,896,900,902,914,928,930,941,943,949,951,954,958,964,967,968,974,985,991,993,996,1015,1016,1023,1025,1030,1037,1040,1054,1063,1084,1085,1094,1108,1118,1124,1139,1141,1153],\\\"gsm8k_cot_llama\\\":[3,5,37,55,57,58,76,81,90,107,124,133,139,141,148,150,169,173,179,190,212,232,244,264,280,286,298,303,310,317,321,332,336,338,346,380,398,417,422,424,427,438,444,449,470,476,498,502,526,533,535,547,561,566,569,571,573,576,593,601,609,611,623,624,630,641,653,657,664,668,677,678,679,684,695,709,712,714,715,727,750,756,761,768,779,780,801,805,808,815,819,820,840,864,878,886,889,902,914,936,938,958,971,974,978,982,983,994,1004,1022,1032,1034,1037,1042,1045,1055,1056,1062,1076,1079,1088,1090,1093,1095,1100,1122,1150,1152,1155,1157,1159,1163,1183,1187,1193,1194,1198,1203,1204,1206,1211,1227,1229,1233,1249,1269,1275,1286,1294,1307],\\\"mmlu_llama_abstract_algebra\\\":[52],\\\"mmlu_llama_anatomy\\\":[43],\\\"mmlu_llama_astronomy\\\":[56],\\\"mmlu_llama_business_ethics\\\":[94],\\\"mmlu_llama_clinical_knowledge\\\":[1,93,182,244,254],\\\"mmlu_llama_college_biology\\\":[59,107],\\\"mmlu_llama_college_chemistry\\\":[16],\\\"mmlu_llama_college_mathematics\\\":[29,65],\\\"mmlu_llama_college_medicine\\\":[27,50,117],\\\"mmlu_llama_college_physics\\\":[37],\\\"mmlu_llama_conceptual_physics\\\":[14,63,109,167,196],\\\"mmlu_llama_electrical_engineering\\\":[53],\\\"mmlu_llama_elementary_mathematics\\\":[16,95,105,252,358],\\\"mmlu_llama_global_facts\\\":[22],\\\"mmlu_llama_high_school_biology\\\":[103,159,182,185,211,219],\\\"mmlu_llama_high_school_chemistry\\\":[34,106,162],\\\"mmlu_llama_high_school_computer_science\\\":[36,53,65,88],\\\"mmlu_llama_high_school_geography\\\":[30,133],\\\"mmlu_llama_high_school_government_and_politics\\\":[89,191],\\\"mmlu_llama_high_school_macroeconomics\\\":[77,78,127,335,354],\\\"mmlu_llama_high_school_mathematics\\\":[106,130,220,243,251],\\\"mmlu_llama_high_school_microeconomics\\\":[175,227],\\\"mmlu_llama_high_school_physics\\\":[63,67,74],\\\"mmlu_llama_high_school_psychology\\\":[237,260,290,348,392],\\\"mmlu_llama_high_school_statistics\\\":[61,71,143],\\\"mmlu_llama_high_school_us_history\\\":[23,149,167,194],\\\"mmlu_llama_high_school_world_history\\\":[71,93,175,195,200,233],\\\"mmlu_llama_human_aging\\\":[0,2,22,100,133],\\\"mmlu_llama_human_sexuality\\\":[113],\\\"mmlu_llama_international_law\\\":[55],\\\"mmlu_llama_jurisprudence\\\":[14,79],\\\"mmlu_llama_logical_fallacies\\\":[15,19,22,35,127],\\\"mmlu_llama_management\\\":[39],\\\"mmlu_llama_marketing\\\":[184,220],\\\"mmlu_llama_medical_genetics\\\":[11,57],\\\"mmlu_llama_miscellaneous\\\":[101,205,245,284,305,364,483,498,581,613,689],\\\"mmlu_llama_moral_disputes\\\":[55,259],\\\"mmlu_llama_moral_scenarios\\\":[27,42,86,155,172,235,288,320,392,403,438,458,466,624,843],\\\"mmlu_llama_nutrition\\\":[95,144,198,212,255,282],\\\"mmlu_llama_philosophy\\\":[10,79,96,114,151,182,200,206,242],\\\"mmlu_llama_prehistory\\\":[1,104,139,186],\\\"mmlu_llama_professional_accounting\\\":[127,180],\\\"mmlu_llama_professional_law\\\":[31,39,210,416,424,428,440,442,459,658,724,753,754,766,813,1026,1036,1059,1234,1279,1313,1336,1339,1428,1454,1522],\\\"mmlu_llama_professional_medicine\\\":[11,201,206],\\\"mmlu_llama_professional_psychology\\\":[62,164,172,266,316,414,579],\\\"mmlu_llama_public_relations\\\":[107],\\\"mmlu_llama_security_studies\\\":[5,46,74,152,169],\\\"mmlu_llama_sociology\\\":[163,167],\\\"mmlu_llama_us_foreign_policy\\\":[58,60,74],\\\"mmlu_llama_virology\\\":[29,66],\\\"mmlu_llama_world_religions\\\":[7,59,118]} --gen_kwargs temperature=0,max_gen_toks=1024 --seed 1234 --use_cache C:\\quantdiff\\results\\tasks\\20261022T060918_gemma-4-12b--unsloth--ud-q2_k_xl\\cache\\responses --output_path C:\\quantdiff\\results\\tasks\\20261022T060918_gemma-4-12b--unsloth--ud-q2_k_xl\\lm_eval --log_samples"
  ],
  "quant": {
    "repo": "unsloth/gemma-4-12b-it-GGUF",
    "file": "gemma-4-12b-it-UD-Q2_K_XL.gguf",
    "revision": "fc034cfff751157913579611efad8462ac1be606",
    "sha256": "aa6e1ccf5cd1c5340c4786f0bda2478181327f14aba2359e51e455d02c90bff3"
  },
  "started_at": "2026-10-22T03:09:17-04:00",
  "finished_at": "2026-10-22T03:18:45-04:00",
  "seconds": {
    "download": 0,
    "eval": 547,
    "total": 568
  },
  "results": {
    "gsm8k_cot_llama": {
      "metric": "exact_match,strict-match",
      "value": 71.33,
      "stderr": 3.7,
      "n": 150,
      "correct": 107,
      "flexible_extract": 71.33
    },
    "arc_challenge_chat": {
      "metric": "letter_match (quantdiff regex on logged samples)",
      "value": 63.33,
      "stderr": 3.95,
      "n": 150,
      "correct": 95
    },
    "mmlu_llama": {
      "metric": "letter_match (quantdiff regex on logged samples)",
      "value": 63.0,
      "stderr": 3.42,
      "n": 200,
      "correct": 126
    }
  },
  "lm_eval_results_file": "lm_eval\\quant\\results_2026-10-22T03-18-44.929472.json"
}