[
 {
  "id": "qwen3.5-9b--mlx-community--mlx-4bit",
  "model_id": "qwen3.5-9b",
  "repo": "mlx-community/Qwen3.5-9B-4bit",
  "revision": "8b2b98c00a6b4d291155e4890773ca8f769aee53",
  "quant": "4bit",
  "size_bytes": 5977071067,
  "machine": {
   "name": "minipro",
   "hardware": "Apple M2 Pro (Mac mini), 12-core CPU, 19-core GPU, 32 GB unified memory",
   "backend": "MLX (Metal)",
   "os": "macOS 27.0.1"
  },
  "software": {
   "lm_eval_version": "0.4.13",
   "python": "3.12.15",
   "mlx": "0.32.4",
   "mlx_lm": "0.32.2",
   "server": "mlx_lm.server (mlx-lm 0.32.2)"
  },
  "finished_at": "2026-10-21T22:01:21-04:00",
  "tasks": {
   "gsm8k": {
    "metric": "exact_match,strict-match",
    "value": 90.67,
    "stderr": 2.38,
    "n": 150,
    "correct": 136,
    "flexible_extract": 90.67,
    "task": "gsm8k_cot_llama"
   }
  },
  "tasks_seconds": {
   "download": 0,
   "eval": 2786,
   "total": 2790
  },
  "toolcall": {
   "score": 93.5,
   "passed": 187,
   "total": 200,
   "request_errors": 0,
   "by_category": {
    "simple_python": {
     "passed": 59,
     "total": 60,
     "score": 98.3
    },
    "multiple": {
     "passed": 38,
     "total": 40,
     "score": 95.0
    },
    "parallel": {
     "passed": 30,
     "total": 30,
     "score": 100.0
    },
    "parallel_multiple": {
     "passed": 26,
     "total": 30,
     "score": 86.7
    },
    "irrelevance": {
     "passed": 34,
     "total": 40,
     "score": 85.0
    }
   }
  },
  "toolcall_error": null,
  "speed": {
   "pp_tps": 254.24,
   "tg_tps": 37.91,
   "peak_memory_gb": 5.928,
   "pp_tps_trials": [
    254.326,
    254.174,
    254.207,
    254.155,
    254.361
   ],
   "tg_tps_trials": [
    37.919,
    37.92,
    37.902,
    37.914,
    37.916
   ],
   "settings": {
    "prompt_tokens": 512,
    "generation_tokens": 128,
    "num_trials": 5,
    "batch_size": 1
   },
   "command": "/Users/minipro/quantdiff/mlx-venv/bin/python -m mlx_lm benchmark --model /Users/minipro/quantdiff/mlx-models/mlx-community__Qwen3.5-9B-4bit -p 512 -g 128 -n 5 -b 1"
  },
  "kld_error": null,
  "logs": {
   "tasks.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/tasks.json",
   "toolcall.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/toolcall.json",
   "speed.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/speed.json",
   "speed.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/speed.log",
   "kld.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/kld.json",
   "kld.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/kld.log",
   "server.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/server.log",
   "result.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/result.json",
   "toolcall.jsonl.gz": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/toolcall.jsonl.gz",
   "samples.jsonl.gz": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-4bit/samples.jsonl.gz"
  },
  "tasks_held": "held: MLX ARC-Challenge and MMLU letter scores are not published. lm-eval stops generation at the first '.' or blank line; under mlx_lm.server the model often explains before giving the letter, so the reply is cut before any letter (ARC: 119-134 of 150 replies, MMLU: 16-20 of 200). Raw answers are in samples.jsonl.gz. GGUF runs are unaffected.",
  "kld": {
   "method": "mlx full-vocab KLD vs BF16 MLX (reference log-probs stored as float16)",
   "positions": 25500,
   "mean_kld": 0.10095768421888351,
   "median_kld": 0.044465504586696625,
   "p99_kld": 0.9747713804244995,
   "max_kld": 17.139354705810547,
   "same_top_pct": 86.75686274509803,
   "ppl_quant": 7.9290554904392625,
   "ppl_ref": 7.258659638050345,
   "ctx": 512,
   "chunks": 100,
   "vocab": 248320,
   "tokens_forwarded": 51200,
   "forward_passes": 100
  }
 },
 {
  "id": "qwen3.5-9b--mlx-community--mlx-5bit",
  "model_id": "qwen3.5-9b",
  "repo": "mlx-community/Qwen3.5-9B-5bit",
  "revision": "64b670725ad59d26c7b77430a3a4ae671afb776a",
  "quant": "5bit",
  "size_bytes": 7096163574,
  "machine": {
   "name": "minipro",
   "hardware": "Apple M2 Pro (Mac mini), 12-core CPU, 19-core GPU, 32 GB unified memory",
   "backend": "MLX (Metal)",
   "os": "macOS 27.0.1"
  },
  "software": {
   "lm_eval_version": "0.4.13",
   "python": "3.12.15",
   "mlx": "0.32.4",
   "mlx_lm": "0.32.2",
   "server": "mlx_lm.server (mlx-lm 0.32.2)"
  },
  "finished_at": "2026-10-21T23:12:35-04:00",
  "tasks": {
   "gsm8k": {
    "metric": "exact_match,strict-match",
    "value": 92.0,
    "stderr": 2.22,
    "n": 150,
    "correct": 138,
    "flexible_extract": 92.0,
    "task": "gsm8k_cot_llama"
   }
  },
  "tasks_seconds": {
   "download": 0,
   "eval": 2884,
   "total": 2887
  },
  "toolcall": {
   "score": 91.0,
   "passed": 182,
   "total": 200,
   "request_errors": 0,
   "by_category": {
    "simple_python": {
     "passed": 58,
     "total": 60,
     "score": 96.7
    },
    "multiple": {
     "passed": 36,
     "total": 40,
     "score": 90.0
    },
    "parallel": {
     "passed": 29,
     "total": 30,
     "score": 96.7
    },
    "parallel_multiple": {
     "passed": 27,
     "total": 30,
     "score": 90.0
    },
    "irrelevance": {
     "passed": 32,
     "total": 40,
     "score": 80.0
    }
   }
  },
  "toolcall_error": null,
  "speed": {
   "pp_tps": 247.3,
   "tg_tps": 31.42,
   "peak_memory_gb": 6.992,
   "pp_tps_trials": [
    247.489,
    247.356,
    247.345,
    247.057,
    247.261
   ],
   "tg_tps_trials": [
    31.424,
    31.409,
    31.429,
    31.432,
    31.424
   ],
   "settings": {
    "prompt_tokens": 512,
    "generation_tokens": 128,
    "num_trials": 5,
    "batch_size": 1
   },
   "command": "/Users/minipro/quantdiff/mlx-venv/bin/python -m mlx_lm benchmark --model /Users/minipro/quantdiff/mlx-models/mlx-community__Qwen3.5-9B-5bit -p 512 -g 128 -n 5 -b 1"
  },
  "kld_error": null,
  "logs": {
   "tasks.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/tasks.json",
   "toolcall.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/toolcall.json",
   "speed.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/speed.json",
   "speed.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/speed.log",
   "kld.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/kld.json",
   "kld.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/kld.log",
   "server.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/server.log",
   "result.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/result.json",
   "toolcall.jsonl.gz": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/toolcall.jsonl.gz",
   "samples.jsonl.gz": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-5bit/samples.jsonl.gz"
  },
  "tasks_held": "held: MLX ARC-Challenge and MMLU letter scores are not published. lm-eval stops generation at the first '.' or blank line; under mlx_lm.server the model often explains before giving the letter, so the reply is cut before any letter (ARC: 119-134 of 150 replies, MMLU: 16-20 of 200). Raw answers are in samples.jsonl.gz. GGUF runs are unaffected.",
  "kld": {
   "method": "mlx full-vocab KLD vs BF16 MLX (reference log-probs stored as float16)",
   "positions": 25500,
   "mean_kld": 0.032942574471235275,
   "median_kld": 0.01173768937587738,
   "p99_kld": 0.2959308326244354,
   "max_kld": 15.576461791992188,
   "same_top_pct": 92.7607843137255,
   "ppl_quant": 7.350662710017511,
   "ppl_ref": 7.258659638050345,
   "ctx": 512,
   "chunks": 100,
   "vocab": 248320,
   "tokens_forwarded": 51200,
   "forward_passes": 100
  }
 },
 {
  "id": "qwen3.5-9b--mlx-community--mlx-6bit",
  "model_id": "qwen3.5-9b",
  "repo": "mlx-community/Qwen3.5-9B-6bit",
  "revision": "76fe4065e622cf34990d3c13ef80ec8531c9a0f7",
  "quant": "6bit",
  "size_bytes": 8215256416,
  "machine": {
   "name": "minipro",
   "hardware": "Apple M2 Pro (Mac mini), 12-core CPU, 19-core GPU, 32 GB unified memory",
   "backend": "MLX (Metal)",
   "os": "macOS 27.0.1"
  },
  "software": {
   "lm_eval_version": "0.4.13",
   "python": "3.12.15",
   "mlx": "0.32.4",
   "mlx_lm": "0.32.2",
   "server": "mlx_lm.server (mlx-lm 0.32.2)"
  },
  "finished_at": "2026-10-22T00:29:27-04:00",
  "tasks": {
   "gsm8k": {
    "metric": "exact_match,strict-match",
    "value": 92.0,
    "stderr": 2.22,
    "n": 150,
    "correct": 138,
    "flexible_extract": 92.67,
    "task": "gsm8k_cot_llama"
   }
  },
  "tasks_seconds": {
   "download": 0,
   "eval": 3025,
   "total": 3029
  },
  "toolcall": {
   "score": 91.5,
   "passed": 183,
   "total": 200,
   "request_errors": 0,
   "by_category": {
    "simple_python": {
     "passed": 58,
     "total": 60,
     "score": 96.7
    },
    "multiple": {
     "passed": 36,
     "total": 40,
     "score": 90.0
    },
    "parallel": {
     "passed": 30,
     "total": 30,
     "score": 100.0
    },
    "parallel_multiple": {
     "passed": 26,
     "total": 30,
     "score": 86.7
    },
    "irrelevance": {
     "passed": 33,
     "total": 40,
     "score": 82.5
    }
   }
  },
  "toolcall_error": null,
  "speed": {
   "pp_tps": 245.27,
   "tg_tps": 26.11,
   "peak_memory_gb": 8.03,
   "pp_tps_trials": [
    245.263,
    245.329,
    245.327,
    245.285,
    245.149
   ],
   "tg_tps_trials": [
    26.09,
    26.115,
    26.11,
    26.119,
    26.111
   ],
   "settings": {
    "prompt_tokens": 512,
    "generation_tokens": 128,
    "num_trials": 5,
    "batch_size": 1
   },
   "command": "/Users/minipro/quantdiff/mlx-venv/bin/python -m mlx_lm benchmark --model /Users/minipro/quantdiff/mlx-models/mlx-community__Qwen3.5-9B-6bit -p 512 -g 128 -n 5 -b 1"
  },
  "kld_error": null,
  "logs": {
   "tasks.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/tasks.json",
   "toolcall.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/toolcall.json",
   "speed.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/speed.json",
   "speed.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/speed.log",
   "kld.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/kld.json",
   "kld.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/kld.log",
   "server.log": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/server.log",
   "result.json": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/result.json",
   "toolcall.jsonl.gz": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/toolcall.jsonl.gz",
   "samples.jsonl.gz": "/logs/mlx/qwen3.5-9b--mlx-community--mlx-6bit/samples.jsonl.gz"
  },
  "tasks_held": "held: MLX ARC-Challenge and MMLU letter scores are not published. lm-eval stops generation at the first '.' or blank line; under mlx_lm.server the model often explains before giving the letter, so the reply is cut before any letter (ARC: 119-134 of 150 replies, MMLU: 16-20 of 200). Raw answers are in samples.jsonl.gz. GGUF runs are unaffected.",
  "kld": {
   "method": "mlx full-vocab KLD vs BF16 MLX (reference log-probs stored as float16)",
   "positions": 25500,
   "mean_kld": 0.009602926671504974,
   "median_kld": 0.0035379407927393913,
   "p99_kld": 0.08057275414466858,
   "max_kld": 9.149763107299805,
   "same_top_pct": 96.10980392156863,
   "ppl_quant": 7.322081408478335,
   "ppl_ref": 7.258659638050345,
   "ctx": 512,
   "chunks": 100,
   "vocab": 248320,
   "tokens_forwarded": 51200,
   "forward_passes": 100
  }
 }
]