{
	"methodology_version": "0.6",
	"drafts": false,
	"methodology_url": "https://www.xybrid.ai/leaderboard/methodology#rankings",
	"data_url": "https://www.xybrid.ai/leaderboard/data.json",
	"objectives": [
		{
			"id": "balanced",
			"label": "Balanced",
			"summary": "Decode speed, time to first token and peak memory, weighted equally.",
			"metrics": [
				"decode_tps",
				"ttft_ms",
				"peak_memory_bytes"
			]
		},
		{
			"id": "speed",
			"label": "Speed",
			"summary": "Decode speed: tokens generated per second.",
			"metrics": [
				"decode_tps"
			]
		},
		{
			"id": "memory",
			"label": "Memory",
			"summary": "Peak memory while generating: the less, the better.",
			"metrics": [
				"peak_memory_bytes"
			]
		}
	],
	"baskets": [
		{
			"id": "desktop",
			"platform": "desktop",
			"devices": [
				{
					"id": "glenn-m5-max-64gb",
					"os": "macOS 27"
				}
			],
			"models": [
				"lfm2.5-1.2b-instruct",
				"qwen3.8-27b"
			],
			"quant_classes": [
				"8-bit"
			],
			"workloads": [
				"archive-256-128",
				"archive-2048-128"
			]
		},
		{
			"id": "mobile",
			"platform": "mobile",
			"devices": [
				{
					"id": "glenn-iphone-12",
					"os": "iOS 27"
				},
				{
					"id": "glenn-oneplus-13",
					"os": "Android 16"
				},
				{
					"id": "glenn-pixel-8",
					"os": "Android 16"
				}
			],
			"models": [
				"lfm2.5-350m",
				"lfm2.5-1.2b-instruct"
			],
			"quant_classes": [
				"8-bit"
			],
			"workloads": [
				"archive-256-128",
				"archive-2048-128"
			]
		}
	],
	"summaries": [
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 4 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 4,
			"metrics": {
				"decode_tps": {
					"n": 4,
					"median": 42.284,
					"mean": 42.28,
					"stddev": 0.285,
					"min": 41.936,
					"max": 42.618,
					"p25": 42.139,
					"p75": 42.425
				},
				"ttft_ms": {
					"n": 4,
					"median": 7783.978,
					"mean": 7870.162,
					"stddev": 183.404,
					"min": 7767.687,
					"max": 8145.003,
					"p25": 7777.708,
					"p75": 7876.433
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 4,
					"median": 265.32,
					"mean": 262.496,
					"stddev": 6.074,
					"min": 253.398,
					"max": 265.944,
					"p25": 262.207,
					"p75": 265.609
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 4 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 4,
			"metrics": {
				"decode_tps": {
					"n": 4,
					"median": 32.217,
					"mean": 31.437,
					"stddev": 3.192,
					"min": 26.941,
					"max": 34.371,
					"p25": 30.512,
					"p75": 33.142
				},
				"ttft_ms": {
					"n": 4,
					"median": 19335.718,
					"mean": 18574.244,
					"stddev": 1707.698,
					"min": 16020.978,
					"max": 19604.562,
					"p25": 18458.743,
					"p75": 19451.219
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 4,
					"median": 105.918,
					"mean": 111.036,
					"stddev": 11.227,
					"min": 104.468,
					"max": 127.84,
					"p25": 105.292,
					"p75": 111.661
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 34.863,
					"mean": 34.869,
					"stddev": 0.182,
					"min": 34.636,
					"max": 35.075,
					"p25": 34.741,
					"p75": 35.024
				},
				"ttft_ms": {
					"n": 6,
					"median": 8775.305,
					"mean": 8800.561,
					"stddev": 82.513,
					"min": 8728.27,
					"max": 8932.058,
					"p25": 8735.516,
					"p75": 8846.458
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 233.421,
					"mean": 232.766,
					"stddev": 2.168,
					"min": 229.331,
					"max": 234.675,
					"p25": 231.542,
					"p75": 234.482
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 45.352,
					"mean": 45.32,
					"stddev": 0.661,
					"min": 44.449,
					"max": 45.987,
					"p25": 44.866,
					"p75": 45.897
				},
				"ttft_ms": {
					"n": 6,
					"median": 1073.437,
					"mean": 1080.733,
					"stddev": 28.512,
					"min": 1054.972,
					"max": 1117.372,
					"p25": 1056.429,
					"p75": 1104.174
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 257.656,
					"mean": 257.974,
					"stddev": 0.906,
					"min": 257.079,
					"max": 259.119,
					"p25": 257.32,
					"p75": 258.757
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 42.504,
					"mean": 42.152,
					"stddev": 0.92,
					"min": 40.394,
					"max": 42.79,
					"p25": 42.04,
					"p75": 42.75
				},
				"ttft_ms": {
					"n": 6,
					"median": 2033.419,
					"mean": 2098.807,
					"stddev": 149.888,
					"min": 1989.582,
					"max": 2372.491,
					"p25": 2001.806,
					"p75": 2137.852
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 125.891,
					"mean": 122.447,
					"stddev": 8.15,
					"min": 107.88,
					"max": 128.643,
					"p25": 119.863,
					"p75": 127.866
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 37.565,
					"mean": 37.585,
					"stddev": 0.064,
					"min": 37.535,
					"max": 37.705,
					"p25": 37.544,
					"p75": 37.591
				},
				"ttft_ms": {
					"n": 6,
					"median": 986.632,
					"mean": 988.048,
					"stddev": 3.82,
					"min": 983.834,
					"max": 993.413,
					"p25": 985.887,
					"p75": 990.762
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 259.681,
					"mean": 259.323,
					"stddev": 0.982,
					"min": 257.944,
					"max": 260.414,
					"p25": 258.627,
					"p75": 259.879
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 4.652,
					"mean": 4.652,
					"stddev": 0.014,
					"min": 4.632,
					"max": 4.671,
					"p25": 4.644,
					"p75": 4.662
				},
				"ttft_ms": {
					"n": 6,
					"median": 42788.319,
					"mean": 42793.902,
					"stddev": 29.573,
					"min": 42756.267,
					"max": 42840.124,
					"p25": 42778.373,
					"p75": 42808.573
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 5.984,
					"mean": 5.983,
					"stddev": 0.004,
					"min": 5.977,
					"max": 5.988,
					"p25": 5.981,
					"p75": 5.985
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 2 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 2,
			"metrics": {
				"decode_tps": {
					"n": 2,
					"median": 19.735,
					"mean": 19.735,
					"stddev": 0.291,
					"min": 19.529,
					"max": 19.94,
					"p25": 19.632,
					"p75": 19.838
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 2,
					"median": 213.71,
					"mean": 213.71,
					"stddev": 2.034,
					"min": 212.272,
					"max": 215.149,
					"p25": 212.991,
					"p75": 214.429
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 22.009,
					"mean": 22.02,
					"stddev": 0.184,
					"min": 21.828,
					"max": 22.282,
					"p25": 21.868,
					"p75": 22.129
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 90.192,
					"mean": 90.442,
					"stddev": 1.324,
					"min": 88.965,
					"max": 92.422,
					"p25": 89.525,
					"p75": 91.229
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 4 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 4,
			"metrics": {
				"decode_tps": {
					"n": 4,
					"median": 17.388,
					"mean": 17.276,
					"stddev": 0.475,
					"min": 16.671,
					"max": 17.655,
					"p25": 17.01,
					"p75": 17.654
				},
				"ttft_ms": {
					"n": 4,
					"median": 27146.491,
					"mean": 27517.989,
					"stddev": 1100.935,
					"min": 26673.299,
					"max": 29105.676,
					"p25": 26840.021,
					"p75": 27824.459
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 4,
					"median": 75.509,
					"mean": 74.575,
					"stddev": 2.899,
					"min": 70.422,
					"max": 76.86,
					"p25": 73.705,
					"p75": 76.379
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 52.113,
					"mean": 52.194,
					"stddev": 0.626,
					"min": 51.396,
					"max": 53.317,
					"p25": 52.007,
					"p75": 52.22
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 480.02,
					"mean": 490.398,
					"stddev": 28.987,
					"min": 474.843,
					"max": 549.356,
					"p25": 477.419,
					"p75": 481.203
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 49.707,
					"mean": 48.195,
					"stddev": 2.899,
					"min": 43.064,
					"max": 50.36,
					"p25": 47.195,
					"p75": 49.908
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 169.694,
					"mean": 160.121,
					"stddev": 17.407,
					"min": 137.14,
					"max": 173.127,
					"p25": 145.637,
					"p75": 172.515
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 51.547,
					"mean": 51.856,
					"stddev": 1.649,
					"min": 50.451,
					"max": 55.09,
					"p25": 51.066,
					"p75": 51.582
				},
				"ttft_ms": {
					"n": 6,
					"median": 8116.904,
					"mean": 8011.983,
					"stddev": 268.744,
					"min": 7464.048,
					"max": 8142.245,
					"p25": 8106.482,
					"p75": 8125.82
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 252.609,
					"mean": 256.157,
					"stddev": 9.098,
					"min": 251.781,
					"max": 274.709,
					"p25": 252.323,
					"p75": 252.907
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 31.713,
					"mean": 31.398,
					"stddev": 1.062,
					"min": 29.371,
					"max": 32.307,
					"p25": 31.333,
					"p75": 31.988
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 156.288,
					"mean": 156.412,
					"stddev": 7.282,
					"min": 147.211,
					"max": 165.375,
					"p25": 151.435,
					"p75": 161.694
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 25.876,
					"mean": 25.424,
					"stddev": 1.351,
					"min": 22.687,
					"max": 26.25,
					"p25": 25.81,
					"p75": 26.019
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 97.093,
					"mean": 95.697,
					"stddev": 5.506,
					"min": 85.41,
					"max": 100.338,
					"p25": 94.964,
					"p75": 99.271
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 24.791,
					"mean": 23.933,
					"stddev": 2.741,
					"min": 19.41,
					"max": 26.494,
					"p25": 22.563,
					"p75": 25.916
				},
				"ttft_ms": {
					"n": 6,
					"median": 2328.244,
					"mean": 2375.782,
					"stddev": 138.887,
					"min": 2283.671,
					"max": 2655.258,
					"p25": 2311.834,
					"p75": 2346.745
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 110.896,
					"mean": 108.935,
					"stddev": 5.847,
					"min": 97.235,
					"max": 113.045,
					"p25": 109.94,
					"p75": 111.662
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 76.3,
					"mean": 76.209,
					"stddev": 0.373,
					"min": 75.64,
					"max": 76.552,
					"p25": 75.991,
					"p75": 76.506
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 558.952,
					"mean": 558.829,
					"stddev": 7.299,
					"min": 545.842,
					"max": 566.372,
					"p25": 558.039,
					"p75": 563.579
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 49.774,
					"mean": 49.778,
					"stddev": 0.388,
					"min": 49.234,
					"max": 50.227,
					"p25": 49.551,
					"p75": 50.083
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 145.224,
					"mean": 144.896,
					"stddev": 1.737,
					"min": 142.152,
					"max": 146.897,
					"p25": 143.993,
					"p75": 146.028
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 63.53,
					"mean": 63.558,
					"stddev": 0.514,
					"min": 63.017,
					"max": 64.143,
					"p25": 63.099,
					"p75": 64.014
				},
				"ttft_ms": {
					"n": 6,
					"median": 743.937,
					"mean": 758.701,
					"stddev": 35.366,
					"min": 726.813,
					"max": 812.393,
					"p25": 734.43,
					"p75": 781.384
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 348.07,
					"mean": 342.773,
					"stddev": 16.198,
					"min": 319.47,
					"max": 357.785,
					"p25": 331.234,
					"p75": 355.232
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 52.961,
					"mean": 53.071,
					"stddev": 0.42,
					"min": 52.653,
					"max": 53.745,
					"p25": 52.763,
					"p75": 53.294
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 665.909,
					"mean": 661.532,
					"stddev": 16.528,
					"min": 631.125,
					"max": 675.685,
					"p25": 658.226,
					"p75": 672.65
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 121.357,
					"mean": 121.229,
					"stddev": 1.055,
					"min": 119.249,
					"max": 122.233,
					"p25": 121.212,
					"p75": 121.852
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 1509.768,
					"mean": 1512.419,
					"stddev": 9.52,
					"min": 1502.568,
					"max": 1527.218,
					"p25": 1505.885,
					"p75": 1517.893
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 97.548,
					"mean": 95.956,
					"stddev": 5.451,
					"min": 85.695,
					"max": 101.438,
					"p25": 95.495,
					"p75": 98.412
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 329.058,
					"mean": 330.675,
					"stddev": 25.824,
					"min": 293.242,
					"max": 365.193,
					"p25": 318.004,
					"p75": 347.15
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 203.039,
					"mean": 202.779,
					"stddev": 1.696,
					"min": 199.686,
					"max": 204.509,
					"p25": 202.552,
					"p75": 203.771
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 1523.864,
					"mean": 1544.745,
					"stddev": 70.129,
					"min": 1471.264,
					"max": 1662.338,
					"p25": 1501.505,
					"p75": 1575.78
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 3 of 6 planned trials were valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 3,
			"metrics": {
				"decode_tps": {
					"n": 3,
					"median": 9.867,
					"mean": 10.226,
					"stddev": 0.816,
					"min": 9.652,
					"max": 11.16,
					"p25": 9.759,
					"p75": 10.513
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 3,
					"median": 45.844,
					"mean": 45.75,
					"stddev": 0.428,
					"min": 45.283,
					"max": 46.123,
					"p25": 45.564,
					"p75": 45.983
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 11.792,
					"mean": 11.783,
					"stddev": 0.158,
					"min": 11.594,
					"max": 11.989,
					"p25": 11.653,
					"p75": 11.892
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 43.607,
					"mean": 43.684,
					"stddev": 0.495,
					"min": 43.167,
					"max": 44.357,
					"p25": 43.289,
					"p75": 44.04
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 5 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 5,
			"metrics": {
				"decode_tps": {
					"n": 5,
					"median": 10.215,
					"mean": 10.257,
					"stddev": 0.305,
					"min": 9.894,
					"max": 10.714,
					"p25": 10.113,
					"p75": 10.348
				},
				"ttft_ms": {
					"n": 5,
					"median": 4789.362,
					"mean": 4777.869,
					"stddev": 25.886,
					"min": 4735.866,
					"max": 4802.412,
					"p25": 4771.801,
					"p75": 4789.905
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 5,
					"median": 53.682,
					"mean": 53.805,
					"stddev": 0.25,
					"min": 53.585,
					"max": 54.205,
					"p25": 53.665,
					"p75": 53.885
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 15.02,
					"mean": 16.99,
					"stddev": 3.486,
					"min": 14.923,
					"max": 23.484,
					"p25": 14.967,
					"p75": 17.66
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 51.363,
					"mean": 52.597,
					"stddev": 7.317,
					"min": 43.935,
					"max": 65.102,
					"p25": 48.645,
					"p75": 54.898
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 4 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 4,
			"metrics": {
				"decode_tps": {
					"n": 4,
					"median": 24.546,
					"mean": 24.431,
					"stddev": 0.308,
					"min": 23.989,
					"max": 24.644,
					"p25": 24.336,
					"p75": 24.64
				},
				"ttft_ms": {
					"n": 4,
					"median": 18100.923,
					"mean": 18122.703,
					"stddev": 50.496,
					"min": 18090.865,
					"max": 18198.098,
					"p25": 18097.527,
					"p75": 18126.099
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 4,
					"median": 113.189,
					"mean": 113.063,
					"stddev": 0.312,
					"min": 112.599,
					"max": 113.276,
					"p25": 113.03,
					"p75": 113.223
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 23.804,
					"mean": 24.228,
					"stddev": 0.867,
					"min": 23.437,
					"max": 25.451,
					"p25": 23.697,
					"p75": 24.859
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 53.462,
					"mean": 57.826,
					"stddev": 7.958,
					"min": 51.093,
					"max": 69.36,
					"p25": 53.069,
					"p75": 63.348
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 28.346,
					"mean": 28.176,
					"stddev": 1.064,
					"min": 26.943,
					"max": 29.183,
					"p25": 27.222,
					"p75": 29.131
				},
				"ttft_ms": {
					"n": 6,
					"median": 2024.61,
					"mean": 2031.304,
					"stddev": 166.216,
					"min": 1876.111,
					"max": 2205.764,
					"p25": 1880.721,
					"p75": 2174.13
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 127.431,
					"mean": 127.23,
					"stddev": 10.441,
					"min": 116.45,
					"max": 137,
					"p25": 118.232,
					"p75": 136.786
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 83.47,
					"mean": 83.173,
					"stddev": 1.087,
					"min": 80.995,
					"max": 83.995,
					"p25": 83.443,
					"p75": 83.621
				},
				"ttft_ms": {
					"n": 6,
					"median": 4731.284,
					"mean": 4813.349,
					"stddev": 188.812,
					"min": 4723.074,
					"max": 5197.249,
					"p25": 4728.061,
					"p75": 4760.484
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 432.844,
					"mean": 425.998,
					"stddev": 15.712,
					"min": 394.074,
					"max": 433.623,
					"p25": 430.234,
					"p75": 433.141
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 54.311,
					"mean": 55.13,
					"stddev": 2.723,
					"min": 53.191,
					"max": 60.573,
					"p25": 53.792,
					"p75": 54.659
				},
				"ttft_ms": {
					"n": 6,
					"median": 2326.933,
					"mean": 2326.948,
					"stddev": 2.581,
					"min": 2323.514,
					"max": 2330.015,
					"p25": 2325.156,
					"p75": 2329.03
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 880.607,
					"mean": 880.637,
					"stddev": 0.952,
					"min": 879.488,
					"max": 881.928,
					"p25": 879.907,
					"p75": 881.291
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 130.835,
					"mean": 130.06,
					"stddev": 1.708,
					"min": 126.767,
					"max": 131.198,
					"p25": 129.903,
					"p75": 131.058
				},
				"ttft_ms": {
					"n": 6,
					"median": 384.116,
					"mean": 386.193,
					"stddev": 7.757,
					"min": 379.397,
					"max": 399.658,
					"p25": 380.35,
					"p75": 389.112
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 786.235,
					"mean": 786.389,
					"stddev": 7.769,
					"min": 777.376,
					"max": 795.488,
					"p25": 779.952,
					"p75": 792.917
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 121.941,
					"mean": 119.463,
					"stddev": 4.92,
					"min": 110.339,
					"max": 122.824,
					"p25": 118.333,
					"p75": 122.439
				},
				"ttft_ms": {
					"n": 6,
					"median": 481.22,
					"mean": 496.65,
					"stddev": 42.671,
					"min": 468.612,
					"max": 581.945,
					"p25": 474.854,
					"p75": 490.933
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 532.227,
					"mean": 518.462,
					"stddev": 39.73,
					"min": 439.863,
					"max": 547.009,
					"p25": 521.444,
					"p75": 539.253
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, f16, uzu default.",
				"The experiments here use different 4-bit schemes: 8da4w, Q4_K_M, mirai-m."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 55.983,
					"mean": 55.52,
					"stddev": 1.81,
					"min": 52.113,
					"max": 57.448,
					"p25": 55.46,
					"p75": 56.227
				},
				"ttft_ms": {
					"n": 6,
					"median": 313.592,
					"mean": 313.735,
					"stddev": 1.92,
					"min": 311.599,
					"max": 317.116,
					"p25": 312.573,
					"p75": 314.107
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 818.499,
					"mean": 818.107,
					"stddev": 5.015,
					"min": 809.302,
					"max": 823.702,
					"p25": 817.087,
					"p75": 821.143
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 2 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 2,
			"metrics": {
				"decode_tps": {
					"n": 2,
					"median": 15.534,
					"mean": 15.534,
					"stddev": 0.055,
					"min": 15.496,
					"max": 15.573,
					"p25": 15.515,
					"p75": 15.554
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 2,
					"median": 133.072,
					"mean": 133.072,
					"stddev": 4.667,
					"min": 129.772,
					"max": 136.372,
					"p25": 131.422,
					"p75": 134.722
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 17.911,
					"mean": 17.864,
					"stddev": 0.099,
					"min": 17.731,
					"max": 17.965,
					"p25": 17.786,
					"p75": 17.919
				},
				"ttft_ms": {
					"n": 6,
					"median": 12623.94,
					"mean": 12617.836,
					"stddev": 18.479,
					"min": 12591.469,
					"max": 12634.029,
					"p25": 12604.276,
					"p75": 12632.922
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 162.366,
					"mean": 162.439,
					"stddev": 0.242,
					"min": 162.212,
					"max": 162.795,
					"p25": 162.249,
					"p75": 162.605
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 3 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 3,
			"metrics": {
				"decode_tps": {
					"n": 3,
					"median": 8.175,
					"mean": 8.171,
					"stddev": 0.082,
					"min": 8.087,
					"max": 8.251,
					"p25": 8.131,
					"p75": 8.213
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 3,
					"median": 99.811,
					"mean": 100.326,
					"stddev": 1.176,
					"min": 99.494,
					"max": 101.672,
					"p25": 99.653,
					"p75": 100.741
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 4 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 4,
			"metrics": {
				"decode_tps": {
					"n": 4,
					"median": 7.143,
					"mean": 7.161,
					"stddev": 0.319,
					"min": 6.813,
					"max": 7.546,
					"p25": 6.961,
					"p75": 7.343
				},
				"ttft_ms": {
					"n": 4,
					"median": 3989.807,
					"mean": 4411.975,
					"stddev": 887.879,
					"min": 3925.309,
					"max": 5742.978,
					"p25": 3967.826,
					"p75": 4433.957
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 4,
					"median": 64.543,
					"mean": 59.827,
					"stddev": 10.052,
					"min": 44.763,
					"max": 65.457,
					"p25": 59.503,
					"p75": 64.867
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 15.738,
					"mean": 15.876,
					"stddev": 0.418,
					"min": 15.562,
					"max": 16.649,
					"p25": 15.587,
					"p75": 15.959
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 122.892,
					"mean": 124.72,
					"stddev": 4.831,
					"min": 121.197,
					"max": 134.184,
					"p25": 122.114,
					"p75": 124.696
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 2 of 6 planned trials were valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 2,
			"metrics": {
				"decode_tps": {
					"n": 2,
					"median": 15.051,
					"mean": 15.051,
					"stddev": 1.516,
					"min": 13.979,
					"max": 16.123,
					"p25": 14.515,
					"p75": 15.587
				},
				"ttft_ms": {
					"n": 2,
					"median": 20522.663,
					"mean": 20522.663,
					"stddev": 2368.202,
					"min": 18848.091,
					"max": 22197.234,
					"p25": 19685.377,
					"p75": 21359.948
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 2,
					"median": 100.478,
					"mean": 100.478,
					"stddev": 11.593,
					"min": 92.281,
					"max": 108.676,
					"p25": 96.379,
					"p75": 104.577
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.767,
					"mean": 18.692,
					"stddev": 0.25,
					"min": 18.195,
					"max": 18.884,
					"p25": 18.727,
					"p75": 18.812
				},
				"ttft_ms": {
					"n": 6,
					"median": 1517.124,
					"mean": 1516.712,
					"stddev": 8.986,
					"min": 1504.083,
					"max": 1526.913,
					"p25": 1510.973,
					"p75": 1523.862
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 169.962,
					"mean": 169.832,
					"stddev": 0.846,
					"min": 168.439,
					"max": 170.939,
					"p25": 169.533,
					"p75": 170.216
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 16.139,
					"mean": 16.245,
					"stddev": 0.413,
					"min": 15.815,
					"max": 16.973,
					"p25": 16.009,
					"p75": 16.365
				},
				"ttft_ms": {
					"n": 6,
					"median": 2473.788,
					"mean": 2458.955,
					"stddev": 95.064,
					"min": 2300.102,
					"max": 2561.993,
					"p25": 2421.653,
					"p75": 2523.284
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 103.561,
					"mean": 104.339,
					"stddev": 4.118,
					"min": 99.997,
					"max": 111.386,
					"p25": 101.63,
					"p75": 105.8
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 5 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 5,
			"metrics": {
				"decode_tps": {
					"n": 5,
					"median": 15.984,
					"mean": 15.815,
					"stddev": 0.432,
					"min": 15.044,
					"max": 16.059,
					"p25": 15.971,
					"p75": 16.017
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 5,
					"median": 150.376,
					"mean": 153.2,
					"stddev": 7.072,
					"min": 146.725,
					"max": 161.677,
					"p25": 147.368,
					"p75": 159.853
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 12.903,
					"mean": 12.88,
					"stddev": 0.297,
					"min": 12.411,
					"max": 13.318,
					"p25": 12.785,
					"p75": 12.973
				},
				"ttft_ms": {
					"n": 6,
					"median": 24421.143,
					"mean": 24912.715,
					"stddev": 2070.853,
					"min": 23106.979,
					"max": 28048.189,
					"p25": 23167.28,
					"p75": 26152.418
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 84.141,
					"mean": 82.758,
					"stddev": 6.688,
					"min": 73.083,
					"max": 88.736,
					"p25": 78.4,
					"p75": 88.504
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 2 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 2,
			"metrics": {
				"decode_tps": {
					"n": 2,
					"median": 24.726,
					"mean": 24.726,
					"stddev": 0.034,
					"min": 24.702,
					"max": 24.75,
					"p25": 24.714,
					"p75": 24.738
				},
				"ttft_ms": {
					"n": 2,
					"median": 7790.012,
					"mean": 7790.012,
					"stddev": 114.423,
					"min": 7709.103,
					"max": 7870.921,
					"p25": 7749.558,
					"p75": 7830.467
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 2,
					"median": 265.05,
					"mean": 265.05,
					"stddev": 3.941,
					"min": 262.263,
					"max": 267.837,
					"p25": 263.656,
					"p75": 266.443
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 23.473,
					"mean": 23.475,
					"stddev": 0.081,
					"min": 23.347,
					"max": 23.577,
					"p25": 23.444,
					"p75": 23.525
				},
				"ttft_ms": {
					"n": 6,
					"median": 8222.097,
					"mean": 8256.313,
					"stddev": 139.008,
					"min": 8137.737,
					"max": 8528.964,
					"p25": 8189.423,
					"p75": 8241.861
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 249.131,
					"mean": 248.154,
					"stddev": 4.085,
					"min": 240.167,
					"max": 251.709,
					"p25": 248.529,
					"p75": 250.125
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 33.434,
					"mean": 32.905,
					"stddev": 1.709,
					"min": 30.822,
					"max": 34.993,
					"p25": 31.471,
					"p75": 33.805
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 260.47,
					"mean": 263.526,
					"stddev": 11.25,
					"min": 254.957,
					"max": 285.269,
					"p25": 256.556,
					"p75": 263.67
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 38.497,
					"mean": 38.502,
					"stddev": 0.345,
					"min": 37.996,
					"max": 38.901,
					"p25": 38.316,
					"p75": 38.775
				},
				"ttft_ms": {
					"n": 6,
					"median": 5712.06,
					"mean": 5625.93,
					"stddev": 226.872,
					"min": 5164.228,
					"max": 5750.545,
					"p25": 5700.71,
					"p75": 5717.834
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 359.119,
					"mean": 365.143,
					"stddev": 15.78,
					"min": 356.784,
					"max": 397.282,
					"p25": 358.723,
					"p75": 359.751
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.259,
					"mean": 17.934,
					"stddev": 0.856,
					"min": 16.645,
					"max": 18.841,
					"p25": 17.378,
					"p75": 18.451
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 170.684,
					"mean": 168.48,
					"stddev": 8.063,
					"min": 153.001,
					"max": 174.756,
					"p25": 168.165,
					"p75": 173.495
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 25.677,
					"mean": 25.638,
					"stddev": 0.132,
					"min": 25.385,
					"max": 25.757,
					"p25": 25.632,
					"p75": 25.704
				},
				"ttft_ms": {
					"n": 6,
					"median": 1047.633,
					"mean": 1054.581,
					"stddev": 27.985,
					"min": 1031.874,
					"max": 1109.721,
					"p25": 1040.654,
					"p75": 1051.132
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 259.984,
					"mean": 260.171,
					"stddev": 1.299,
					"min": 258.778,
					"max": 262.341,
					"p25": 259.264,
					"p75": 260.683
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 15.877,
					"mean": 15.573,
					"stddev": 1.27,
					"min": 13.561,
					"max": 16.943,
					"p25": 14.896,
					"p75": 16.427
				},
				"ttft_ms": {
					"n": 6,
					"median": 1923.496,
					"mean": 1927.279,
					"stddev": 58.86,
					"min": 1866.613,
					"max": 2028.136,
					"p25": 1884.044,
					"p75": 1944.154
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 134.624,
					"mean": 134.4,
					"stddev": 3.998,
					"min": 127.581,
					"max": 138.645,
					"p25": 133.25,
					"p75": 137.258
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 24.1,
					"mean": 24.049,
					"stddev": 0.155,
					"min": 23.747,
					"max": 24.181,
					"p25": 24.057,
					"p75": 24.119
				},
				"ttft_ms": {
					"n": 6,
					"median": 938.203,
					"mean": 947.552,
					"stddev": 24.705,
					"min": 935.578,
					"max": 997.925,
					"p25": 937.06,
					"p75": 938.593
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 273.112,
					"mean": 270.557,
					"stddev": 6.764,
					"min": 256.766,
					"max": 273.858,
					"p25": 272.997,
					"p75": 273.43
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 36.353,
					"mean": 36.351,
					"stddev": 0.056,
					"min": 36.251,
					"max": 36.405,
					"p25": 36.343,
					"p75": 36.393
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 332.595,
					"mean": 333.048,
					"stddev": 5.379,
					"min": 325.329,
					"max": 339.709,
					"p25": 330.209,
					"p75": 337.134
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 42.118,
					"mean": 42.144,
					"stddev": 0.246,
					"min": 41.858,
					"max": 42.588,
					"p25": 42.03,
					"p75": 42.166
				},
				"ttft_ms": {
					"n": 6,
					"median": 483.803,
					"mean": 495.925,
					"stddev": 30.112,
					"min": 476.335,
					"max": 555.411,
					"p25": 479.241,
					"p75": 494.809
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 538.906,
					"mean": 526.485,
					"stddev": 30.638,
					"min": 467.132,
					"max": 546.938,
					"p25": 524.526,
					"p75": 545.198
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 54.18,
					"mean": 53.929,
					"stddev": 0.821,
					"min": 52.375,
					"max": 54.591,
					"p25": 53.842,
					"p75": 54.435
				},
				"ttft_ms": {
					"n": 6,
					"median": 10199.806,
					"mean": 10242.754,
					"stddev": 95.269,
					"min": 10173.854,
					"max": 10422.556,
					"p25": 10186.371,
					"p75": 10258.91
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3439980108,
					"mean": 3440214965.333,
					"stddev": 1272883.804,
					"min": 3439021632,
					"max": 3442462296,
					"p25": 3439312478,
					"p75": 3440561812
				},
				"prefill_tps": {
					"n": 6,
					"median": 201.697,
					"mean": 200.857,
					"stddev": 1.851,
					"min": 197.376,
					"max": 202.19,
					"p25": 200.516,
					"p75": 201.968
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 224.131,
					"mean": 224.389,
					"stddev": 0.672,
					"min": 223.668,
					"max": 225.442,
					"p25": 224.014,
					"p75": 224.772
				},
				"ttft_ms": {
					"n": 6,
					"median": 296.672,
					"mean": 296.814,
					"stddev": 0.502,
					"min": 296.409,
					"max": 297.695,
					"p25": 296.437,
					"p75": 296.977
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1992730796,
					"mean": 1992733538.667,
					"stddev": 126776.277,
					"min": 1992525984,
					"max": 1992886456,
					"p25": 1992693968,
					"p75": 1992816830
				},
				"prefill_tps": {
					"n": 6,
					"median": 6965.563,
					"mean": 6961.605,
					"stddev": 12.694,
					"min": 6938.586,
					"max": 6971.511,
					"p25": 6958.354,
					"p75": 6970.734
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 258.332,
					"mean": 258.447,
					"stddev": 0.8,
					"min": 257.488,
					"max": 259.65,
					"p25": 257.913,
					"p75": 258.915
				},
				"ttft_ms": {
					"n": 6,
					"median": 233.675,
					"mean": 233.608,
					"stddev": 2.27,
					"min": 230.566,
					"max": 236.172,
					"p25": 232.099,
					"p75": 235.408
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3383249548,
					"mean": 3383855720,
					"stddev": 48441234.59,
					"min": 3327396432,
					"max": 3438250576,
					"p25": 3343440464,
					"p75": 3426425472
				},
				"prefill_tps": {
					"n": 6,
					"median": 8790.3,
					"mean": 8793.74,
					"stddev": 85.634,
					"min": 8697.942,
					"max": 8908.686,
					"p25": 8725.362,
					"p75": 8851.196
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 258.359,
					"mean": 258.33,
					"stddev": 0.613,
					"min": 257.48,
					"max": 259.331,
					"p25": 258.058,
					"p75": 258.462
				},
				"ttft_ms": {
					"n": 6,
					"median": 208.467,
					"mean": 208.956,
					"stddev": 1.056,
					"min": 207.941,
					"max": 210.602,
					"p25": 208.337,
					"p75": 209.591
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3204950236,
					"mean": 3210632745.333,
					"stddev": 41681203.307,
					"min": 3144550608,
					"max": 3255847120,
					"p25": 3198421206,
					"p75": 3244177616
				},
				"prefill_tps": {
					"n": 6,
					"median": 9857.697,
					"mean": 9835.309,
					"stddev": 49.019,
					"min": 9759.166,
					"max": 9882.691,
					"p25": 9805.58,
					"p75": 9864.222
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 212.662,
					"mean": 212.793,
					"stddev": 2.775,
					"min": 208.701,
					"max": 217.011,
					"p25": 211.756,
					"p75": 213.867
				},
				"ttft_ms": {
					"n": 6,
					"median": 313.343,
					"mean": 313.867,
					"stddev": 1.286,
					"min": 312.599,
					"max": 315.773,
					"p25": 313.025,
					"p75": 314.754
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1994647760,
					"mean": 1994672336,
					"stddev": 178885.507,
					"min": 1994442984,
					"max": 1994918096,
					"p25": 1994561696,
					"p75": 1994795246
				},
				"prefill_tps": {
					"n": 6,
					"median": 6555.177,
					"mean": 6543.801,
					"stddev": 26.777,
					"min": 6503.549,
					"max": 6569.598,
					"p25": 6525.866,
					"p75": 6561.201
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 54.09,
					"mean": 53.895,
					"stddev": 0.619,
					"min": 52.854,
					"max": 54.434,
					"p25": 53.607,
					"p75": 54.363
				},
				"ttft_ms": {
					"n": 6,
					"median": 1590.47,
					"mean": 1593.735,
					"stddev": 12.558,
					"min": 1582.612,
					"max": 1618.353,
					"p25": 1588.298,
					"p75": 1592.315
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2405649276,
					"mean": 2406419320,
					"stddev": 1397423.821,
					"min": 2405288816,
					"max": 2408500056,
					"p25": 2405542786,
					"p75": 2407353224
				},
				"prefill_tps": {
					"n": 6,
					"median": 165.989,
					"mean": 165.667,
					"stddev": 1.259,
					"min": 163.204,
					"max": 166.809,
					"p25": 165.809,
					"p75": 166.194
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 227.982,
					"mean": 228.049,
					"stddev": 0.751,
					"min": 226.917,
					"max": 229.275,
					"p25": 227.957,
					"p75": 228.136
				},
				"ttft_ms": {
					"n": 6,
					"median": 51.899,
					"mean": 52.319,
					"stddev": 1.322,
					"min": 51.256,
					"max": 54.938,
					"p25": 51.739,
					"p75": 52.152
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1974945940,
					"mean": 1977908709.333,
					"stddev": 7353924.112,
					"min": 1974814832,
					"max": 1992919224,
					"p25": 1974864014,
					"p75": 1974978696
				},
				"prefill_tps": {
					"n": 6,
					"median": 5074.008,
					"mean": 5034.271,
					"stddev": 119.893,
					"min": 4797.781,
					"max": 5132.318,
					"p25": 5045.372,
					"p75": 5087.264
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 265.159,
					"mean": 264.652,
					"stddev": 1.129,
					"min": 263.108,
					"max": 265.558,
					"p25": 263.738,
					"p75": 265.536
				},
				"ttft_ms": {
					"n": 6,
					"median": 58.019,
					"mean": 57.511,
					"stddev": 1.781,
					"min": 55.253,
					"max": 59.286,
					"p25": 55.968,
					"p75": 58.908
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2279516932,
					"mean": 2293653661.333,
					"stddev": 92409783.733,
					"min": 2209303360,
					"max": 2460797760,
					"p25": 2228816788,
					"p75": 2310531916
				},
				"prefill_tps": {
					"n": 6,
					"median": 4456.57,
					"mean": 4499.627,
					"stddev": 143.176,
					"min": 4360.49,
					"max": 4683.584,
					"p25": 4386.556,
					"p75": 4622.139
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 265.15,
					"mean": 265.19,
					"stddev": 0.361,
					"min": 264.795,
					"max": 265.626,
					"p25": 264.898,
					"p75": 265.491
				},
				"ttft_ms": {
					"n": 6,
					"median": 48.536,
					"mean": 48.511,
					"stddev": 0.202,
					"min": 48.216,
					"max": 48.784,
					"p25": 48.394,
					"p75": 48.621
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2013170148,
					"mean": 1999980948,
					"stddev": 93778692.63,
					"min": 1901807776,
					"max": 2155465128,
					"p25": 1929697716,
					"p75": 2014091724
				},
				"prefill_tps": {
					"n": 6,
					"median": 5340.736,
					"mean": 5342.574,
					"stddev": 22.242,
					"min": 5312.746,
					"max": 5376.118,
					"p25": 5330.227,
					"p75": 5353.974
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 4-bit schemes: litert-int4, Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 218.378,
					"mean": 217.091,
					"stddev": 3.23,
					"min": 212.33,
					"max": 220.373,
					"p25": 214.857,
					"p75": 219.147
				},
				"ttft_ms": {
					"n": 6,
					"median": 62.525,
					"mean": 62.731,
					"stddev": 1.578,
					"min": 61.208,
					"max": 65.567,
					"p25": 61.671,
					"p75": 63.013
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1975945400,
					"mean": 1978924565.333,
					"stddev": 7434582.977,
					"min": 1975732408,
					"max": 1994098920,
					"p25": 1975842976,
					"p75": 1975998672
				},
				"prefill_tps": {
					"n": 6,
					"median": 4141.222,
					"mean": 4131.443,
					"stddev": 102.891,
					"min": 3949.947,
					"max": 4236.657,
					"p25": 4110.313,
					"p75": 4200.007
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 60.314,
					"mean": 60.327,
					"stddev": 0.376,
					"min": 59.815,
					"max": 60.882,
					"p25": 60.113,
					"p75": 60.522
				},
				"ttft_ms": {
					"n": 6,
					"median": 2922.9,
					"mean": 2924.133,
					"stddev": 6.157,
					"min": 2916.758,
					"max": 2931.65,
					"p25": 2920.023,
					"p75": 2929.369
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 4750881172,
					"mean": 4751383618.667,
					"stddev": 966606.923,
					"min": 4750643592,
					"max": 4753068472,
					"p25": 4750803360,
					"p75": 4751757704
				},
				"prefill_tps": {
					"n": 6,
					"median": 711.858,
					"mean": 711.599,
					"stddev": 1.481,
					"min": 709.837,
					"max": 713.378,
					"p25": 710.301,
					"p75": 712.624
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 153.566,
					"mean": 153.591,
					"stddev": 0.202,
					"min": 153.29,
					"max": 153.869,
					"p25": 153.511,
					"p75": 153.711
				},
				"ttft_ms": {
					"n": 6,
					"median": 282.966,
					"mean": 283.21,
					"stddev": 0.676,
					"min": 282.529,
					"max": 284.307,
					"p25": 282.78,
					"p75": 283.57
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3193416716,
					"mean": 3193310224,
					"stddev": 367825.447,
					"min": 3192704000,
					"max": 3193703448,
					"p25": 3193134104,
					"p75": 3193539602
				},
				"prefill_tps": {
					"n": 6,
					"median": 7306.315,
					"mean": 7300.085,
					"stddev": 16.128,
					"min": 7275.337,
					"max": 7317.265,
					"p25": 7290.106,
					"p75": 7309.508
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 160.047,
					"mean": 160.05,
					"stddev": 0.476,
					"min": 159.399,
					"max": 160.818,
					"p25": 159.822,
					"p75": 160.191
				},
				"ttft_ms": {
					"n": 6,
					"median": 241.952,
					"mean": 241.557,
					"stddev": 0.832,
					"min": 239.939,
					"max": 242.094,
					"p25": 241.525,
					"p75": 242.003
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 4440673796,
					"mean": 4456334165.333,
					"stddev": 55027916.604,
					"min": 4401647096,
					"max": 4528180752,
					"p25": 4413681102,
					"p75": 4501777960
				},
				"prefill_tps": {
					"n": 6,
					"median": 8490.316,
					"mean": 8503.447,
					"stddev": 29.052,
					"min": 8484.196,
					"max": 8559.94,
					"p25": 8487.395,
					"p75": 8504.699
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 160.362,
					"mean": 160.082,
					"stddev": 0.644,
					"min": 158.831,
					"max": 160.507,
					"p25": 160.05,
					"p75": 160.454
				},
				"ttft_ms": {
					"n": 6,
					"median": 221.162,
					"mean": 221.112,
					"stddev": 0.39,
					"min": 220.467,
					"max": 221.507,
					"p25": 220.95,
					"p75": 221.411
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 4302826604,
					"mean": 4285126410.667,
					"stddev": 57686183.228,
					"min": 4210281568,
					"max": 4342320200,
					"p25": 4236557414,
					"p75": 4329233504
				},
				"prefill_tps": {
					"n": 6,
					"median": 9292.262,
					"mean": 9294.123,
					"stddev": 15.783,
					"min": 9277.379,
					"max": 9320.996,
					"p25": 9283.225,
					"p75": 9299.285
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 142.394,
					"mean": 143.096,
					"stddev": 2.828,
					"min": 140.454,
					"max": 147.323,
					"p25": 140.761,
					"p75": 144.943
				},
				"ttft_ms": {
					"n": 6,
					"median": 298.354,
					"mean": 298.291,
					"stddev": 0.585,
					"min": 297.366,
					"max": 298.92,
					"p25": 298.001,
					"p75": 298.738
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3195227232,
					"mean": 3195276368,
					"stddev": 236971.39,
					"min": 3194948656,
					"max": 3195669552,
					"p25": 3195214944,
					"p75": 3195337824
				},
				"prefill_tps": {
					"n": 6,
					"median": 6884.65,
					"mean": 6886.389,
					"stddev": 13.146,
					"min": 6873.59,
					"max": 6907.577,
					"p25": 6875.668,
					"p75": 6892.554
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 70.189,
					"mean": 70.134,
					"stddev": 0.831,
					"min": 68.81,
					"max": 71.001,
					"p25": 69.775,
					"p75": 70.78
				},
				"ttft_ms": {
					"n": 6,
					"median": 2356.265,
					"mean": 2363.721,
					"stddev": 18.588,
					"min": 2345.547,
					"max": 2394.727,
					"p25": 2352.652,
					"p75": 2372.62
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 895.537,
					"mean": 895.882,
					"stddev": 1.492,
					"min": 894.222,
					"max": 897.808,
					"p25": 894.801,
					"p75": 897.109
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 48.331,
					"mean": 48.448,
					"stddev": 1.074,
					"min": 47.006,
					"max": 49.712,
					"p25": 47.79,
					"p75": 49.358
				},
				"ttft_ms": {
					"n": 6,
					"median": 2209.554,
					"mean": 2209.17,
					"stddev": 3.472,
					"min": 2202.956,
					"max": 2212.339,
					"p25": 2208.47,
					"p75": 2211.77
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 927.426,
					"mean": 927.591,
					"stddev": 1.451,
					"min": 926.261,
					"max": 930.191,
					"p25": 926.511,
					"p75": 927.884
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 60.346,
					"mean": 60.276,
					"stddev": 0.221,
					"min": 59.985,
					"max": 60.502,
					"p25": 60.097,
					"p75": 60.433
				},
				"ttft_ms": {
					"n": 6,
					"median": 540.676,
					"mean": 540.716,
					"stddev": 1.15,
					"min": 539.133,
					"max": 542.363,
					"p25": 540.09,
					"p75": 541.333
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3673206432,
					"mean": 3673446694.667,
					"stddev": 759431.354,
					"min": 3672723056,
					"max": 3674459784,
					"p25": 3672845948,
					"p75": 3674070616
				},
				"prefill_tps": {
					"n": 6,
					"median": 515.903,
					"mean": 515.938,
					"stddev": 1.099,
					"min": 514.381,
					"max": 517.36,
					"p25": 515.327,
					"p75": 516.686
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 155.103,
					"mean": 155.104,
					"stddev": 0.102,
					"min": 154.981,
					"max": 155.23,
					"p25": 155.021,
					"p75": 155.183
				},
				"ttft_ms": {
					"n": 6,
					"median": 52.692,
					"mean": 52.632,
					"stddev": 0.311,
					"min": 52.132,
					"max": 52.907,
					"p25": 52.483,
					"p75": 52.891
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3175738392,
					"mean": 3178641090.667,
					"stddev": 7066551.868,
					"min": 3175681048,
					"max": 3193064472,
					"p25": 3175701510,
					"p75": 3175885866
				},
				"prefill_tps": {
					"n": 6,
					"median": 4994.985,
					"mean": 4997.45,
					"stddev": 34.623,
					"min": 4956.917,
					"max": 5051.103,
					"p25": 4974.086,
					"p75": 5013.439
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 162.632,
					"mean": 162.528,
					"stddev": 0.816,
					"min": 161.108,
					"max": 163.55,
					"p25": 162.373,
					"p75": 162.876
				},
				"ttft_ms": {
					"n": 6,
					"median": 61.243,
					"mean": 60.681,
					"stddev": 1.367,
					"min": 58.901,
					"max": 61.89,
					"p25": 59.518,
					"p75": 61.708
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3523325600,
					"mean": 3535589008,
					"stddev": 81037783.416,
					"min": 3463794144,
					"max": 3662646992,
					"p25": 3468238490,
					"p75": 3573755594
				},
				"prefill_tps": {
					"n": 6,
					"median": 4219.749,
					"mean": 4260.867,
					"stddev": 97.949,
					"min": 4175.093,
					"max": 4388.597,
					"p25": 4187.441,
					"p75": 4343.943
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 163.104,
					"mean": 162.13,
					"stddev": 2.224,
					"min": 158.12,
					"max": 163.857,
					"p25": 161.468,
					"p75": 163.533
				},
				"ttft_ms": {
					"n": 6,
					"median": 53.242,
					"mean": 53.538,
					"stddev": 0.799,
					"min": 53.04,
					"max": 55.12,
					"p25": 53.069,
					"p75": 53.491
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3244035276,
					"mean": 3246266282.667,
					"stddev": 29771400.767,
					"min": 3209514272,
					"max": 3294022944,
					"p25": 3227765988,
					"p75": 3258744096
				},
				"prefill_tps": {
					"n": 6,
					"median": 4867.993,
					"mean": 4839.201,
					"stddev": 68.573,
					"min": 4703.289,
					"max": 4881.583,
					"p25": 4842.754,
					"p75": 4877.004
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, model-dtype.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0, mlx-affine-8bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 148.832,
					"mean": 143.63,
					"stddev": 13.529,
					"min": 116.035,
					"max": 150.115,
					"p25": 148.695,
					"p75": 149.196
				},
				"ttft_ms": {
					"n": 6,
					"median": 62.764,
					"mean": 62.702,
					"stddev": 0.913,
					"min": 61.171,
					"max": 63.72,
					"p25": 62.411,
					"p75": 63.316
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3176836180,
					"mean": 3179867212,
					"stddev": 7493021.309,
					"min": 3176696904,
					"max": 3195161672,
					"p25": 3176791094,
					"p75": 3176881248
				},
				"prefill_tps": {
					"n": 6,
					"median": 4126.337,
					"mean": 4132.177,
					"stddev": 63.777,
					"min": 4062.138,
					"max": 4241.149,
					"p25": 4090.532,
					"p75": 4150.462
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 74.816,
					"mean": 74.786,
					"stddev": 0.556,
					"min": 73.92,
					"max": 75.365,
					"p25": 74.511,
					"p75": 75.245
				},
				"ttft_ms": {
					"n": 6,
					"median": 382.726,
					"mean": 384.31,
					"stddev": 9.164,
					"min": 372.584,
					"max": 398.975,
					"p25": 379.645,
					"p75": 388.357
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 789.445,
					"mean": 792.077,
					"stddev": 7.917,
					"min": 783.277,
					"max": 802.644,
					"p25": 787.092,
					"p75": 798.367
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, uzu default.",
				"The experiments here use different 8-bit schemes: Q8_0, mirai-l."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 49.838,
					"mean": 49.398,
					"stddev": 1.076,
					"min": 47.727,
					"max": 50.315,
					"p25": 48.718,
					"p75": 50.201
				},
				"ttft_ms": {
					"n": 6,
					"median": 297.492,
					"mean": 296.647,
					"stddev": 3.094,
					"min": 292.832,
					"max": 299.644,
					"p25": 293.896,
					"p75": 299.168
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 862.861,
					"mean": 865.546,
					"stddev": 9.121,
					"min": 857.024,
					"max": 876.667,
					"p25": 858.019,
					"p75": 873.806
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 38.913,
					"mean": 38.898,
					"stddev": 0.146,
					"min": 38.723,
					"max": 39.098,
					"p25": 38.781,
					"p75": 38.981
				},
				"ttft_ms": {
					"n": 6,
					"median": 2242.363,
					"mean": 2240.543,
					"stddev": 4.56,
					"min": 2234.398,
					"max": 2244.858,
					"p25": 2236.834,
					"p75": 2243.804
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 913.864,
					"mean": 914.592,
					"stddev": 1.848,
					"min": 912.819,
					"max": 917.158,
					"p25": 913.289,
					"p75": 916.03
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mirai-m, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 528.67,
					"mean": 528.203,
					"stddev": 1.157,
					"min": 525.919,
					"max": 529.012,
					"p25": 528.246,
					"p75": 528.799
				},
				"ttft_ms": {
					"n": 6,
					"median": 126.265,
					"mean": 126.577,
					"stddev": 1.911,
					"min": 124.383,
					"max": 129.706,
					"p25": 125.383,
					"p75": 127.384
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1018480176,
					"mean": 1019482314.667,
					"stddev": 2469895.2,
					"min": 1018136088,
					"max": 1024460312,
					"p25": 1018197534,
					"p75": 1019045406
				},
				"prefill_tps": {
					"n": 6,
					"median": 31301.114,
					"mean": 31355.346,
					"stddev": 135.893,
					"min": 31245.232,
					"max": 31611.28,
					"p25": 31277.443,
					"p75": 31378.115
				}
			}
		},
		{
			"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-iphone-12--ios-27--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 39.746,
					"mean": 39.682,
					"stddev": 0.299,
					"min": 39.222,
					"max": 39.986,
					"p25": 39.511,
					"p75": 39.904
				},
				"ttft_ms": {
					"n": 6,
					"median": 303.889,
					"mean": 304.2,
					"stddev": 3.341,
					"min": 300.984,
					"max": 309.632,
					"p25": 301.455,
					"p75": 305.594
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 845.096,
					"mean": 844.15,
					"stddev": 9.3,
					"min": 829.007,
					"max": 853.222,
					"p25": 840.267,
					"p75": 851.64
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-4w-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 456.835,
					"mean": 454.695,
					"stddev": 4.729,
					"min": 445.614,
					"max": 458.484,
					"p25": 454.387,
					"p75": 456.835
				},
				"ttft_ms": {
					"n": 6,
					"median": 331.609,
					"mean": 331.384,
					"stddev": 0.82,
					"min": 329.971,
					"max": 332.098,
					"p25": 331.062,
					"p75": 332.004
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 6177.993,
					"mean": 6181.118,
					"stddev": 15.254,
					"min": 6168.675,
					"max": 6206.061,
					"p25": 6168.675,
					"p75": 6187.311
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: 8da4w, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 121.648,
					"mean": 122.024,
					"stddev": 3.934,
					"min": 116.835,
					"max": 128.543,
					"p25": 120.212,
					"p75": 123.216
				},
				"ttft_ms": {
					"n": 6,
					"median": 2405.157,
					"mean": 2509.378,
					"stddev": 267.248,
					"min": 2357.01,
					"max": 3052.065,
					"p25": 2397.19,
					"p75": 2433.048
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 851.559,
					"mean": 822.903,
					"stddev": 74.934,
					"min": 671.255,
					"max": 869.27,
					"p25": 841.794,
					"p75": 854.314
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: litert-int4, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 104.309,
					"mean": 104.918,
					"stddev": 5.519,
					"min": 96.945,
					"max": 113.972,
					"p25": 103.491,
					"p75": 106.144
				},
				"ttft_ms": {
					"n": 6,
					"median": 4587.53,
					"mean": 4667.734,
					"stddev": 228.842,
					"min": 4527.464,
					"max": 5127.054,
					"p25": 4546.834,
					"p75": 4629.549
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2358463092,
					"mean": 2358403021.333,
					"stddev": 2122084.571,
					"min": 2355087952,
					"max": 2360871600,
					"p25": 2357443182,
					"p75": 2359937658
				},
				"prefill_tps": {
					"n": 6,
					"median": 448.938,
					"mean": 441.997,
					"stddev": 20.266,
					"min": 401.472,
					"max": 454.927,
					"p25": 444.878,
					"p75": 452.873
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-gpu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 8/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: litert-int4-gpu, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 349.193,
					"mean": 349.013,
					"stddev": 0.601,
					"min": 347.894,
					"max": 349.549,
					"p25": 348.912,
					"p75": 349.371
				},
				"ttft_ms": {
					"n": 6,
					"median": 374.288,
					"mean": 373.725,
					"stddev": 2.963,
					"min": 369.203,
					"max": 376.775,
					"p25": 372.042,
					"p75": 375.947
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2018387324,
					"mean": 2019190152,
					"stddev": 3622459.471,
					"min": 2015577456,
					"max": 2025850272,
					"p25": 2017146230,
					"p75": 2019751334
				},
				"prefill_tps": {
					"n": 6,
					"median": 5799.472,
					"mean": 5810.331,
					"stddev": 43.293,
					"min": 5772.575,
					"max": 5881.458,
					"p25": 5775.995,
					"p75": 5830.499
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 450.162,
					"mean": 451.537,
					"stddev": 9.669,
					"min": 441.102,
					"max": 466.257,
					"p25": 444.08,
					"p75": 457.154
				},
				"ttft_ms": {
					"n": 6,
					"median": 109.356,
					"mean": 109.138,
					"stddev": 8.208,
					"min": 96.064,
					"max": 119.991,
					"p25": 106.068,
					"p75": 113.656
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1674954360,
					"mean": 1675232856,
					"stddev": 2074786.186,
					"min": 1672365640,
					"max": 1678296624,
					"p25": 1674225218,
					"p75": 1676371576
				},
				"prefill_tps": {
					"n": 6,
					"median": 18729.87,
					"mean": 18856.72,
					"stddev": 1464.114,
					"min": 17067.947,
					"max": 21319.121,
					"p25": 18024.046,
					"p75": 19311.024
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 442.126,
					"mean": 442.685,
					"stddev": 2.153,
					"min": 439.909,
					"max": 445.347,
					"p25": 441.533,
					"p75": 444.481
				},
				"ttft_ms": {
					"n": 6,
					"median": 139.132,
					"mean": 139.316,
					"stddev": 1.261,
					"min": 137.784,
					"max": 141.331,
					"p25": 138.56,
					"p75": 139.895
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1016268204,
					"mean": 1016361062.667,
					"stddev": 770614.646,
					"min": 1015629264,
					"max": 1017726392,
					"p25": 1015809464,
					"p75": 1016530372
				},
				"prefill_tps": {
					"n": 6,
					"median": 15034.688,
					"mean": 15025.026,
					"stddev": 132.884,
					"min": 14809.137,
					"max": 15198.854,
					"p25": 14977.502,
					"p75": 15094.435
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 538.486,
					"mean": 538.828,
					"stddev": 2.101,
					"min": 536.506,
					"max": 541.924,
					"p25": 537.295,
					"p75": 540.125
				},
				"ttft_ms": {
					"n": 6,
					"median": 104.588,
					"mean": 104.432,
					"stddev": 1.388,
					"min": 102.601,
					"max": 105.968,
					"p25": 103.425,
					"p75": 105.504
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2125113072,
					"mean": 2119605321.333,
					"stddev": 20153180.495,
					"min": 2082539248,
					"max": 2136524528,
					"p25": 2115372820,
					"p75": 2133440222
				},
				"prefill_tps": {
					"n": 6,
					"median": 19715.266,
					"mean": 19749.703,
					"stddev": 265.379,
					"min": 19457.323,
					"max": 20104.252,
					"p25": 19545.997,
					"p75": 19941.22
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 533.271,
					"mean": 534.311,
					"stddev": 2.952,
					"min": 531.641,
					"max": 539.59,
					"p25": 532.516,
					"p75": 535.191
				},
				"ttft_ms": {
					"n": 6,
					"median": 97.698,
					"mean": 96.116,
					"stddev": 4.03,
					"min": 87.924,
					"max": 98.36,
					"p25": 97.329,
					"p75": 97.781
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1962370416,
					"mean": 1960300578.667,
					"stddev": 6882824.224,
					"min": 1949795720,
					"max": 1966753160,
					"p25": 1956078978,
					"p75": 1965491550
				},
				"prefill_tps": {
					"n": 6,
					"median": 21107.312,
					"mean": 21489.7,
					"stddev": 970.924,
					"min": 20972.218,
					"max": 23465.787,
					"p25": 21086.178,
					"p75": 21181.659
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 436.715,
					"mean": 434.804,
					"stddev": 12.221,
					"min": 416.329,
					"max": 447.647,
					"p25": 427.731,
					"p75": 444.189
				},
				"ttft_ms": {
					"n": 6,
					"median": 145.392,
					"mean": 145.979,
					"stddev": 1.942,
					"min": 143.868,
					"max": 149.441,
					"p25": 145.035,
					"p75": 146.498
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1017513436,
					"mean": 1017494349.333,
					"stddev": 956980.113,
					"min": 1015940608,
					"max": 1018578384,
					"p25": 1017132580,
					"p75": 1018189312
				},
				"prefill_tps": {
					"n": 6,
					"median": 14181.678,
					"mean": 14130.564,
					"stddev": 193.825,
					"min": 13796.913,
					"max": 14358.73,
					"p25": 14067.079,
					"p75": 14222.05
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mirai-m, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 540.725,
					"mean": 540.635,
					"stddev": 0.666,
					"min": 539.794,
					"max": 541.365,
					"p25": 540.086,
					"p75": 541.176
				},
				"ttft_ms": {
					"n": 6,
					"median": 75.214,
					"mean": 75.515,
					"stddev": 0.742,
					"min": 74.91,
					"max": 76.872,
					"p25": 75.051,
					"p75": 75.717
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 964290060,
					"mean": 975261878.667,
					"stddev": 27025059.664,
					"min": 963888664,
					"max": 1030424112,
					"p25": 964093428,
					"p75": 964560384
				},
				"prefill_tps": {
					"n": 6,
					"median": 13204.809,
					"mean": 13285.173,
					"stddev": 237.678,
					"min": 13069.893,
					"max": 13700.096,
					"p25": 13125.902,
					"p75": 13375.077
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-4w-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 472.119,
					"mean": 472.419,
					"stddev": 2.058,
					"min": 470.37,
					"max": 475.655,
					"p25": 470.808,
					"p75": 473.44
				},
				"ttft_ms": {
					"n": 6,
					"median": 52.272,
					"mean": 52.197,
					"stddev": 0.302,
					"min": 51.625,
					"max": 52.499,
					"p25": 52.183,
					"p75": 52.34
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 4923.077,
					"mean": 4923.684,
					"stddev": 59.907,
					"min": 4830.189,
					"max": 5019.608,
					"p25": 4923.077,
					"p75": 4923.077
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: 8da4w, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 158.752,
					"mean": 158.999,
					"stddev": 1.831,
					"min": 156.597,
					"max": 161.783,
					"p25": 158.009,
					"p75": 159.95
				},
				"ttft_ms": {
					"n": 6,
					"median": 261.89,
					"mean": 263.67,
					"stddev": 4.021,
					"min": 261.176,
					"max": 271.659,
					"p25": 261.618,
					"p75": 263.38
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 977.099,
					"mean": 971.126,
					"stddev": 15.226,
					"min": 941.176,
					"max": 980.843,
					"p25": 971.548,
					"p75": 979.907
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: litert-int4, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 103.549,
					"mean": 103.368,
					"stddev": 0.432,
					"min": 102.703,
					"max": 103.823,
					"p25": 103.108,
					"p75": 103.606
				},
				"ttft_ms": {
					"n": 6,
					"median": 716.492,
					"mean": 717.479,
					"stddev": 3.117,
					"min": 715.029,
					"max": 723.638,
					"p25": 715.991,
					"p75": 717.17
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1321625484,
					"mean": 1321688289.333,
					"stddev": 1205777.009,
					"min": 1320208280,
					"max": 1323730816,
					"p25": 1321011090,
					"p75": 1322006406
				},
				"prefill_tps": {
					"n": 6,
					"median": 370.532,
					"mean": 370.066,
					"stddev": 1.629,
					"min": 366.865,
					"max": 371.317,
					"p25": 370.176,
					"p75": 370.951
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-gpu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 8/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: litert-int4-gpu, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 330.838,
					"mean": 331.91,
					"stddev": 4.494,
					"min": 327.581,
					"max": 340.191,
					"p25": 329.295,
					"p75": 332.631
				},
				"ttft_ms": {
					"n": 6,
					"median": 113.745,
					"mean": 112.995,
					"stddev": 4.911,
					"min": 104.388,
					"max": 117.413,
					"p25": 111.652,
					"p75": 116.73
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1959109964,
					"mean": 1958935197.333,
					"stddev": 2339237.156,
					"min": 1956234560,
					"max": 1961297240,
					"p25": 1956971810,
					"p75": 1960977764
				},
				"prefill_tps": {
					"n": 6,
					"median": 3437.895,
					"mean": 3437.984,
					"stddev": 51.203,
					"min": 3380.475,
					"max": 3497.22,
					"p25": 3393.861,
					"p75": 3480.903
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 444.531,
					"mean": 443.195,
					"stddev": 3.433,
					"min": 437.075,
					"max": 445.997,
					"p25": 442.034,
					"p75": 445.506
				},
				"ttft_ms": {
					"n": 6,
					"median": 26.593,
					"mean": 26.525,
					"stddev": 0.229,
					"min": 26.199,
					"max": 26.834,
					"p25": 26.383,
					"p75": 26.611
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1672955452,
					"mean": 1672327402.667,
					"stddev": 4941973.218,
					"min": 1664009776,
					"max": 1677706776,
					"p25": 1670506062,
					"p75": 1675724384
				},
				"prefill_tps": {
					"n": 6,
					"median": 9626.776,
					"mean": 9651.934,
					"stddev": 83.493,
					"min": 9540.136,
					"max": 9771.365,
					"p25": 9620.082,
					"p75": 9703.218
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 454.57,
					"mean": 454.939,
					"stddev": 2.328,
					"min": 452.316,
					"max": 458.762,
					"p25": 453.421,
					"p75": 455.926
				},
				"ttft_ms": {
					"n": 6,
					"median": 30.595,
					"mean": 30.016,
					"stddev": 3.247,
					"min": 23.854,
					"max": 33.624,
					"p25": 30.528,
					"p75": 30.843
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 999703980,
					"mean": 1002281725.333,
					"stddev": 7264848.689,
					"min": 998131104,
					"max": 1017054600,
					"p25": 999343526,
					"p75": 999830980
				},
				"prefill_tps": {
					"n": 6,
					"median": 8802.877,
					"mean": 9104.688,
					"stddev": 1181.392,
					"min": 7993.505,
					"max": 11422.962,
					"p25": 8763.243,
					"p75": 8842.624
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 575.776,
					"mean": 567.038,
					"stddev": 22.002,
					"min": 523.442,
					"max": 582.916,
					"p25": 568.848,
					"p75": 577.28
				},
				"ttft_ms": {
					"n": 6,
					"median": 34.774,
					"mean": 35.026,
					"stddev": 0.94,
					"min": 34.174,
					"max": 36.607,
					"p25": 34.318,
					"p75": 35.439
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1200400184,
					"mean": 1206451381.333,
					"stddev": 17393577.654,
					"min": 1187891048,
					"max": 1228015512,
					"p25": 1194800976,
					"p75": 1221900136
				},
				"prefill_tps": {
					"n": 6,
					"median": 7492.668,
					"mean": 7443.056,
					"stddev": 202.855,
					"min": 7106.176,
					"max": 7621.997,
					"p25": 7349.363,
					"p75": 7605.592
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 582.879,
					"mean": 583.145,
					"stddev": 5.513,
					"min": 574.939,
					"max": 589.697,
					"p25": 580.472,
					"p75": 587.323
				},
				"ttft_ms": {
					"n": 6,
					"median": 28.872,
					"mean": 28.91,
					"stddev": 0.226,
					"min": 28.639,
					"max": 29.267,
					"p25": 28.773,
					"p75": 29.022
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 959833576,
					"mean": 955611937.333,
					"stddev": 12424826.761,
					"min": 930383336,
					"max": 962315632,
					"p25": 959350230,
					"p75": 961545686
				},
				"prefill_tps": {
					"n": 6,
					"median": 9023.939,
					"mean": 9010.901,
					"stddev": 70.81,
					"min": 8896.921,
					"max": 9091.555,
					"p25": 8977.319,
					"p75": 9056.442
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 438.066,
					"mean": 438,
					"stddev": 5.129,
					"min": 430.598,
					"max": 443.667,
					"p25": 434.989,
					"p75": 442.248
				},
				"ttft_ms": {
					"n": 6,
					"median": 34.23,
					"mean": 34.185,
					"stddev": 0.652,
					"min": 33.501,
					"max": 34.852,
					"p25": 33.593,
					"p75": 34.747
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1000613340,
					"mean": 1003237518.667,
					"stddev": 7566515.729,
					"min": 998098384,
					"max": 1018512920,
					"p25": 1000252892,
					"p75": 1001244124
				},
				"prefill_tps": {
					"n": 6,
					"median": 7671.102,
					"mean": 7693.29,
					"stddev": 188.211,
					"min": 7514.383,
					"max": 7898.797,
					"p25": 7526.535,
					"p75": 7862.281
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mirai-l, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 368.763,
					"mean": 368.756,
					"stddev": 0.184,
					"min": 368.52,
					"max": 369.003,
					"p25": 368.628,
					"p75": 368.87
				},
				"ttft_ms": {
					"n": 6,
					"median": 127.426,
					"mean": 127.916,
					"stddev": 1.56,
					"min": 126.269,
					"max": 130.253,
					"p25": 126.871,
					"p75": 128.934
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1619363808,
					"mean": 1619639581.333,
					"stddev": 948581.762,
					"min": 1618839472,
					"max": 1621510040,
					"p25": 1619224520,
					"p75": 1619527654
				},
				"prefill_tps": {
					"n": 6,
					"median": 30504.806,
					"mean": 30501.185,
					"stddev": 81.661,
					"min": 30394.325,
					"max": 30624.757,
					"p25": 30448.79,
					"p75": 30537.425
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: litert-int8, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 118.786,
					"mean": 118.554,
					"stddev": 0.825,
					"min": 117.028,
					"max": 119.263,
					"p25": 118.402,
					"p75": 119.088
				},
				"ttft_ms": {
					"n": 6,
					"median": 1493.277,
					"mean": 1492.009,
					"stddev": 5.459,
					"min": 1483.949,
					"max": 1498.555,
					"p25": 1488.543,
					"p75": 1495.342
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2890886108,
					"mean": 2891000796,
					"stddev": 1691188.122,
					"min": 2888895440,
					"max": 2893237224,
					"p25": 2889845730,
					"p75": 2892172246
				},
				"prefill_tps": {
					"n": 6,
					"median": 1396.168,
					"mean": 1397.168,
					"stddev": 5.263,
					"min": 1390.825,
					"max": 1404.796,
					"p25": 1393.804,
					"p75": 1400.566
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 329.622,
					"mean": 329.668,
					"stddev": 0.854,
					"min": 328.414,
					"max": 330.877,
					"p25": 329.282,
					"p75": 330.135
				},
				"ttft_ms": {
					"n": 6,
					"median": 96.322,
					"mean": 96.318,
					"stddev": 0.162,
					"min": 96.112,
					"max": 96.499,
					"p25": 96.198,
					"p75": 96.451
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2039268744,
					"mean": 2039825820,
					"stddev": 2375865.99,
					"min": 2037745056,
					"max": 2044380552,
					"p25": 2038490522,
					"p75": 2039862718
				},
				"prefill_tps": {
					"n": 6,
					"median": 21262.017,
					"mean": 21263.024,
					"stddev": 35.803,
					"min": 21223.018,
					"max": 21308.473,
					"p25": 21233.537,
					"p75": 21289.435
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 321.372,
					"mean": 321.545,
					"stddev": 0.51,
					"min": 321.102,
					"max": 322.406,
					"p25": 321.168,
					"p75": 321.783
				},
				"ttft_ms": {
					"n": 6,
					"median": 133.041,
					"mean": 132.996,
					"stddev": 1.004,
					"min": 131.631,
					"max": 134.595,
					"p25": 132.481,
					"p75": 133.291
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1531234104,
					"mean": 1531496252,
					"stddev": 749824.656,
					"min": 1530644256,
					"max": 1532577616,
					"p25": 1531086642,
					"p75": 1531995984
				},
				"prefill_tps": {
					"n": 6,
					"median": 15731.827,
					"mean": 15723.931,
					"stddev": 123.098,
					"min": 15519.619,
					"max": 15885.081,
					"p25": 15688.252,
					"p75": 15784.087
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 340.577,
					"mean": 340.9,
					"stddev": 0.89,
					"min": 339.967,
					"max": 342.395,
					"p25": 340.409,
					"p75": 341.291
				},
				"ttft_ms": {
					"n": 6,
					"median": 105.061,
					"mean": 104.133,
					"stddev": 2.01,
					"min": 100.186,
					"max": 105.309,
					"p25": 104.126,
					"p75": 105.291
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2726086876,
					"mean": 2723230630.667,
					"stddev": 5372063.955,
					"min": 2714405000,
					"max": 2727168448,
					"p25": 2720512328,
					"p75": 2726758548
				},
				"prefill_tps": {
					"n": 6,
					"median": 19612.679,
					"mean": 19797.57,
					"stddev": 394.429,
					"min": 19564.574,
					"max": 20575.263,
					"p25": 19581.409,
					"p75": 19790.098
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 338.445,
					"mean": 337.817,
					"stddev": 1.285,
					"min": 335.924,
					"max": 338.86,
					"p25": 336.913,
					"p75": 338.732
				},
				"ttft_ms": {
					"n": 6,
					"median": 100.69,
					"mean": 100.648,
					"stddev": 0.288,
					"min": 100.149,
					"max": 100.92,
					"p25": 100.556,
					"p75": 100.866
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2549237536,
					"mean": 2542872352,
					"stddev": 17385398.817,
					"min": 2520713016,
					"max": 2558281528,
					"p25": 2527909628,
					"p75": 2556532512
				},
				"prefill_tps": {
					"n": 6,
					"median": 20476.329,
					"mean": 20483.948,
					"stddev": 55.282,
					"min": 20431.17,
					"max": 20577.124,
					"p25": 20440.301,
					"p75": 20504.916
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 319.553,
					"mean": 319.514,
					"stddev": 3.795,
					"min": 315.935,
					"max": 323.159,
					"p25": 316.032,
					"p75": 322.91
				},
				"ttft_ms": {
					"n": 6,
					"median": 139.482,
					"mean": 139.48,
					"stddev": 0.604,
					"min": 138.839,
					"max": 140.471,
					"p25": 139.024,
					"p75": 139.67
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1533921188,
					"mean": 1533896608,
					"stddev": 343181.65,
					"min": 1533495168,
					"max": 1534412720,
					"p25": 1533634456,
					"p75": 1534048176
				},
				"prefill_tps": {
					"n": 6,
					"median": 14798.886,
					"mean": 14789.63,
					"stddev": 60.41,
					"min": 14690.797,
					"max": 14846.82,
					"p25": 14762.071,
					"p75": 14839.166
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mirai-l, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 374.271,
					"mean": 374.171,
					"stddev": 0.522,
					"min": 373.373,
					"max": 374.719,
					"p25": 373.862,
					"p75": 374.568
				},
				"ttft_ms": {
					"n": 6,
					"median": 75.074,
					"mean": 75.184,
					"stddev": 0.726,
					"min": 74.269,
					"max": 76.489,
					"p25": 74.957,
					"p75": 75.228
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1565534092,
					"mean": 1576074501.333,
					"stddev": 25935626.233,
					"min": 1565312944,
					"max": 1629013984,
					"p25": 1565329322,
					"p75": 1565726646
				},
				"prefill_tps": {
					"n": 6,
					"median": 13385.291,
					"mean": 13516.5,
					"stddev": 480.414,
					"min": 13101.331,
					"max": 14425.786,
					"p25": 13230.963,
					"p75": 13562.657
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: litert-int8, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 119.058,
					"mean": 118.842,
					"stddev": 0.745,
					"min": 117.592,
					"max": 119.565,
					"p25": 118.503,
					"p75": 119.361
				},
				"ttft_ms": {
					"n": 6,
					"median": 276.502,
					"mean": 276.538,
					"stddev": 0.765,
					"min": 275.637,
					"max": 277.626,
					"p25": 275.955,
					"p75": 277.017
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1858160860,
					"mean": 1858316504,
					"stddev": 947371.878,
					"min": 1857390824,
					"max": 1859602664,
					"p25": 1857509584,
					"p75": 1859008708
				},
				"prefill_tps": {
					"n": 6,
					"median": 1014.827,
					"mean": 1014.801,
					"stddev": 3.653,
					"min": 1009.519,
					"max": 1019.798,
					"p25": 1012.967,
					"p75": 1016.824
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 330.068,
					"mean": 329.55,
					"stddev": 1.372,
					"min": 327.308,
					"max": 330.88,
					"p25": 328.829,
					"p75": 330.435
				},
				"ttft_ms": {
					"n": 6,
					"median": 25.475,
					"mean": 25.475,
					"stddev": 0.115,
					"min": 25.323,
					"max": 25.629,
					"p25": 25.402,
					"p75": 25.547
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2035910012,
					"mean": 2034222472,
					"stddev": 5183750.891,
					"min": 2027505032,
					"max": 2040284552,
					"p25": 2029962626,
					"p75": 2037286298
				},
				"prefill_tps": {
					"n": 6,
					"median": 10049.331,
					"mean": 10049.305,
					"stddev": 45.543,
					"min": 9988.685,
					"max": 10109.387,
					"p25": 10020.844,
					"p75": 10078.145
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 326.154,
					"mean": 321.958,
					"stddev": 13.512,
					"min": 294.837,
					"max": 332.246,
					"p25": 325.703,
					"p75": 326.64
				},
				"ttft_ms": {
					"n": 6,
					"median": 30.457,
					"mean": 29.908,
					"stddev": 2.964,
					"min": 24.122,
					"max": 32.321,
					"p25": 30.239,
					"p75": 31.556
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1514211140,
					"mean": 1517354133.333,
					"stddev": 7606522.886,
					"min": 1513490208,
					"max": 1532806992,
					"p25": 1513858878,
					"p75": 1515300682
				},
				"prefill_tps": {
					"n": 6,
					"median": 8827.206,
					"mean": 9106.707,
					"stddev": 1100.543,
					"min": 8335.775,
					"max": 11301.929,
					"p25": 8529.41,
					"p75": 8895.287
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 361.94,
					"mean": 361.889,
					"stddev": 1.929,
					"min": 358.787,
					"max": 364.767,
					"p25": 361.51,
					"p75": 362.386
				},
				"ttft_ms": {
					"n": 6,
					"median": 33.509,
					"mean": 33.811,
					"stddev": 1.978,
					"min": 31.346,
					"max": 37.349,
					"p25": 33.11,
					"p75": 34.008
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1812957428,
					"mean": 1814314609.333,
					"stddev": 7592605.921,
					"min": 1806354760,
					"max": 1824049456,
					"p25": 1808099626,
					"p75": 1820555526
				},
				"prefill_tps": {
					"n": 6,
					"median": 7774.798,
					"mean": 7730.031,
					"stddev": 442.959,
					"min": 6967.693,
					"max": 8327.641,
					"p25": 7666.001,
					"p75": 7872.841
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 361.06,
					"mean": 361.105,
					"stddev": 1.158,
					"min": 359.686,
					"max": 362.632,
					"p25": 360.239,
					"p75": 361.937
				},
				"ttft_ms": {
					"n": 6,
					"median": 30.683,
					"mean": 30.761,
					"stddev": 0.22,
					"min": 30.578,
					"max": 31.167,
					"p25": 30.623,
					"p75": 30.812
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1549141864,
					"mean": 1547424306.667,
					"stddev": 5157977.949,
					"min": 1537148800,
					"max": 1551615896,
					"p25": 1548474234,
					"p75": 1549219760
				},
				"prefill_tps": {
					"n": 6,
					"median": 8485.534,
					"mean": 8464.739,
					"stddev": 62.655,
					"min": 8347.735,
					"max": 8513.185,
					"p25": 8453.409,
					"p75": 8506.69
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 325.026,
					"mean": 319.008,
					"stddev": 15.574,
					"min": 288.056,
					"max": 329.84,
					"p25": 320.447,
					"p75": 326.641
				},
				"ttft_ms": {
					"n": 6,
					"median": 34.547,
					"mean": 34.828,
					"stddev": 0.977,
					"min": 33.99,
					"max": 36.565,
					"p25": 34.137,
					"p75": 35.124
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1515784112,
					"mean": 1519088202.667,
					"stddev": 7628053.403,
					"min": 1515407256,
					"max": 1534592896,
					"p25": 1515624356,
					"p75": 1516988330
				},
				"prefill_tps": {
					"n": 6,
					"median": 7574.549,
					"mean": 7511.983,
					"stddev": 210.247,
					"min": 7148.841,
					"max": 7696.933,
					"p25": 7431.213,
					"p75": 7663.832
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype.",
				"Its weights are 4-bit in another scheme: mlx-4w-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 738.372,
					"mean": 737.719,
					"stddev": 7.348,
					"min": 725.714,
					"max": 747.059,
					"p25": 735.171,
					"p75": 741.611
				},
				"ttft_ms": {
					"n": 6,
					"median": 106.736,
					"mean": 106.726,
					"stddev": 0.162,
					"min": 106.526,
					"max": 106.97,
					"p25": 106.61,
					"p75": 106.802
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 19140.187,
					"mean": 19200.376,
					"stddev": 93.245,
					"min": 19140.187,
					"max": 19320.755,
					"p25": 19140.187,
					"p75": 19275.613
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-et-xnnpack-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype.",
				"Its weights are 4-bit in another scheme: 8da4w, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 246.842,
					"mean": 245.47,
					"stddev": 4.132,
					"min": 237.383,
					"max": 248.532,
					"p25": 245.531,
					"p75": 247.806
				},
				"ttft_ms": {
					"n": 6,
					"median": 720.466,
					"mean": 720.997,
					"stddev": 1.928,
					"min": 719.312,
					"max": 724.546,
					"p25": 719.732,
					"p75": 721.397
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 2844.444,
					"mean": 2841.827,
					"stddev": 6.886,
					"min": 2828.729,
					"max": 2848.401,
					"p25": 2841.486,
					"p75": 2844.444
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype.",
				"Its weights are 4-bit in another scheme: litert-int4, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 184.619,
					"mean": 184.038,
					"stddev": 1.583,
					"min": 180.979,
					"max": 185.332,
					"p25": 183.961,
					"p75": 184.86
				},
				"ttft_ms": {
					"n": 6,
					"median": 1498.685,
					"mean": 1500.158,
					"stddev": 3.575,
					"min": 1497.502,
					"max": 1506.659,
					"p25": 1497.709,
					"p75": 1501.196
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1175340748,
					"mean": 1175272477.333,
					"stddev": 720699.04,
					"min": 1174341336,
					"max": 1176225472,
					"p25": 1174730432,
					"p75": 1175729862
				},
				"prefill_tps": {
					"n": 6,
					"median": 1381.942,
					"mean": 1380.609,
					"stddev": 3.37,
					"min": 1374.412,
					"max": 1383.076,
					"p25": 1379.745,
					"p75": 1382.939
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 884.919,
					"mean": 887.512,
					"stddev": 9.671,
					"min": 879.294,
					"max": 906.599,
					"p25": 883.378,
					"p75": 886.086
				},
				"ttft_ms": {
					"n": 6,
					"median": 46.484,
					"mean": 47.014,
					"stddev": 1.092,
					"min": 46.266,
					"max": 49.054,
					"p25": 46.366,
					"p75": 47.224
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1131202132,
					"mean": 1132714921.333,
					"stddev": 2915387.439,
					"min": 1130251872,
					"max": 1136674376,
					"p25": 1130825294,
					"p75": 1134995034
				},
				"prefill_tps": {
					"n": 6,
					"median": 44058.171,
					"mean": 43580.256,
					"stddev": 985.734,
					"min": 41749.908,
					"max": 44265.768,
					"p25": 43371.081,
					"p75": 44170.144
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 700.573,
					"mean": 703.366,
					"stddev": 11.817,
					"min": 692.997,
					"max": 726.582,
					"p25": 698.205,
					"p75": 701.683
				},
				"ttft_ms": {
					"n": 6,
					"median": 64.06,
					"mean": 62.41,
					"stddev": 4.057,
					"min": 54.144,
					"max": 64.37,
					"p25": 63.725,
					"p75": 64.173
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 501318748,
					"mean": 501395190.667,
					"stddev": 391536.236,
					"min": 500991032,
					"max": 501974120,
					"p25": 501085258,
					"p75": 501650488
				},
				"prefill_tps": {
					"n": 6,
					"median": 33581.224,
					"mean": 34627.211,
					"stddev": 2622.767,
					"min": 33461.865,
					"max": 39978.917,
					"p25": 33513.353,
					"p75": 33647.286
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 946.793,
					"mean": 946.319,
					"stddev": 2.777,
					"min": 941.445,
					"max": 949.462,
					"p25": 945.591,
					"p75": 947.871
				},
				"ttft_ms": {
					"n": 6,
					"median": 57.551,
					"mean": 57.164,
					"stddev": 1.014,
					"min": 55.243,
					"max": 57.927,
					"p25": 57.019,
					"p75": 57.789
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1966597548,
					"mean": 2009228716,
					"stddev": 224820943.774,
					"min": 1691452856,
					"max": 2328135072,
					"p25": 1915762086,
					"p75": 2144478642
				},
				"prefill_tps": {
					"n": 6,
					"median": 35995.284,
					"mean": 36240.253,
					"stddev": 662.566,
					"min": 35755.438,
					"max": 37488.559,
					"p25": 35803.263,
					"p75": 36349.592
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 897.818,
					"mean": 897.912,
					"stddev": 2.034,
					"min": 894.801,
					"max": 901.157,
					"p25": 897.533,
					"p75": 898.285
				},
				"ttft_ms": {
					"n": 6,
					"median": 56.21,
					"mean": 54.754,
					"stddev": 3.857,
					"min": 46.896,
					"max": 56.763,
					"p25": 56.11,
					"p75": 56.33
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1907278892,
					"mean": 1809886942.667,
					"stddev": 507710243.01,
					"min": 848315472,
					"max": 2322957368,
					"p25": 1804821542,
					"p75": 2053936178
				},
				"prefill_tps": {
					"n": 6,
					"median": 36803.472,
					"mean": 38002.67,
					"stddev": 3066.967,
					"min": 36471.133,
					"max": 44255.246,
					"p25": 36758.503,
					"p75": 36905.254
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 649.956,
					"mean": 657.218,
					"stddev": 20.031,
					"min": 639.886,
					"max": 689.21,
					"p25": 642.003,
					"p75": 668.703
				},
				"ttft_ms": {
					"n": 6,
					"median": 65.226,
					"mean": 64.482,
					"stddev": 5.453,
					"min": 54.512,
					"max": 70.803,
					"p25": 64.155,
					"p75": 66.802
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 503350412,
					"mean": 502875272,
					"stddev": 1245625.06,
					"min": 500892776,
					"max": 504054936,
					"p25": 502154392,
					"p75": 503723136
				},
				"prefill_tps": {
					"n": 6,
					"median": 31927.739,
					"mean": 32551.665,
					"stddev": 3129.407,
					"min": 29289.361,
					"max": 38465.873,
					"p25": 31198.804,
					"p75": 32539.526
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mirai-m, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 1000.879,
					"mean": 1000.655,
					"stddev": 2.216,
					"min": 996.579,
					"max": 1003.152,
					"p25": 1000.589,
					"p75": 1001.684
				},
				"ttft_ms": {
					"n": 6,
					"median": 67.911,
					"mean": 67.858,
					"stddev": 0.54,
					"min": 67.167,
					"max": 68.719,
					"p25": 67.521,
					"p75": 68.016
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 519070956,
					"mean": 526973497.333,
					"stddev": 19725327.344,
					"min": 518309064,
					"max": 567231760,
					"p25": 518902996,
					"p75": 519251168
				},
				"prefill_tps": {
					"n": 6,
					"median": 19884.269,
					"mean": 19964.888,
					"stddev": 483.263,
					"min": 19380.725,
					"max": 20824.86,
					"p25": 19762.843,
					"p75": 20040.697
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-4w-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 786.385,
					"mean": 785.584,
					"stddev": 3.956,
					"min": 779.141,
					"max": 788.82,
					"p25": 783.951,
					"p75": 788.82
				},
				"ttft_ms": {
					"n": 6,
					"median": 23.383,
					"mean": 23.353,
					"stddev": 0.134,
					"min": 23.124,
					"max": 23.516,
					"p25": 23.313,
					"p75": 23.415
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 11130.435,
					"mean": 11053.14,
					"stddev": 189.333,
					"min": 10666.667,
					"max": 11130.435,
					"p25": 11130.435,
					"p75": 11130.435
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-et-xnnpack-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: 8da4w, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 335.536,
					"mean": 336.437,
					"stddev": 2.32,
					"min": 334.211,
					"max": 340.483,
					"p25": 335.092,
					"p75": 337.319
				},
				"ttft_ms": {
					"n": 6,
					"median": 85.834,
					"mean": 86.043,
					"stddev": 0.404,
					"min": 85.742,
					"max": 86.695,
					"p25": 85.772,
					"p75": 86.258
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 2976.744,
					"mean": 2977.013,
					"stddev": 30.964,
					"min": 2942.529,
					"max": 3011.765,
					"p25": 2951.083,
					"p75": 3003.01
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: litert-int4, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 183.489,
					"mean": 183.258,
					"stddev": 2.132,
					"min": 179.405,
					"max": 185.583,
					"p25": 182.98,
					"p75": 184.452
				},
				"ttft_ms": {
					"n": 6,
					"median": 244.058,
					"mean": 245.607,
					"stddev": 4.085,
					"min": 243.615,
					"max": 253.927,
					"p25": 243.741,
					"p75": 244.275
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 668919116,
					"mean": 669167598.667,
					"stddev": 906890.132,
					"min": 668468544,
					"max": 670893376,
					"p25": 668562752,
					"p75": 669250886
				},
				"prefill_tps": {
					"n": 6,
					"median": 1118.456,
					"mean": 1111.072,
					"stddev": 19.102,
					"min": 1072.283,
					"max": 1122.059,
					"p25": 1116.547,
					"p75": 1119.065
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 911.895,
					"mean": 911.27,
					"stddev": 3.976,
					"min": 904.913,
					"max": 915.903,
					"p25": 909.496,
					"p75": 913.712
				},
				"ttft_ms": {
					"n": 6,
					"median": 18.715,
					"mean": 20.558,
					"stddev": 3.057,
					"min": 18.388,
					"max": 25.027,
					"p25": 18.591,
					"p75": 22.643
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1127818824,
					"mean": 1129844974.667,
					"stddev": 3792781.481,
					"min": 1127171704,
					"max": 1136215576,
					"p25": 1127269960,
					"p75": 1131673142
				},
				"prefill_tps": {
					"n": 6,
					"median": 13679.163,
					"mean": 12664.9,
					"stddev": 1713.366,
					"min": 10228.953,
					"max": 13922.123,
					"p25": 11429.642,
					"p75": 13769.937
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 742.841,
					"mean": 741.021,
					"stddev": 5.169,
					"min": 734.18,
					"max": 745.699,
					"p25": 736.703,
					"p75": 745.14
				},
				"ttft_ms": {
					"n": 6,
					"median": 21.445,
					"mean": 21.459,
					"stddev": 0.103,
					"min": 21.293,
					"max": 21.592,
					"p25": 21.438,
					"p75": 21.517
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 491922524,
					"mean": 493304217.333,
					"stddev": 3941148.113,
					"min": 489980984,
					"max": 501040184,
					"p25": 491639870,
					"p75": 493040708
				},
				"prefill_tps": {
					"n": 6,
					"median": 12794.596,
					"mean": 12800.442,
					"stddev": 43.24,
					"min": 12754.085,
					"max": 12870.142,
					"p25": 12769.991,
					"p75": 12819.229
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 1107.527,
					"mean": 1106.814,
					"stddev": 3.76,
					"min": 1099.996,
					"max": 1110.062,
					"p25": 1106.069,
					"p75": 1109.524
				},
				"ttft_ms": {
					"n": 6,
					"median": 24.499,
					"mean": 24.606,
					"stddev": 0.571,
					"min": 23.889,
					"max": 25.592,
					"p25": 24.371,
					"p75": 24.748
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 774350456,
					"mean": 771231994.667,
					"stddev": 21318430.543,
					"min": 732661320,
					"max": 798213776,
					"p25": 771360226,
					"p75": 776676972
				},
				"prefill_tps": {
					"n": 6,
					"median": 10657.616,
					"mean": 10620.045,
					"stddev": 248.573,
					"min": 10202.048,
					"max": 10948.593,
					"p25": 10551.131,
					"p75": 10718.474
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 1102.388,
					"mean": 1101.757,
					"stddev": 2.68,
					"min": 1097.088,
					"max": 1104.646,
					"p25": 1100.88,
					"p75": 1103.338
				},
				"ttft_ms": {
					"n": 6,
					"median": 23.221,
					"mean": 23.23,
					"stddev": 0.054,
					"min": 23.18,
					"max": 23.322,
					"p25": 23.188,
					"p75": 23.249
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 538330372,
					"mean": 537726862.667,
					"stddev": 1793670.696,
					"min": 534348856,
					"max": 539526416,
					"p25": 537588996,
					"p75": 538445060
				},
				"prefill_tps": {
					"n": 6,
					"median": 11257.944,
					"mean": 11251.635,
					"stddev": 23.774,
					"min": 11206.444,
					"max": 11271.574,
					"p25": 11249.534,
					"p75": 11266.367
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: bf16, fp32, litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 684.932,
					"mean": 688.606,
					"stddev": 17.022,
					"min": 667.817,
					"max": 718.61,
					"p25": 681.757,
					"p75": 692.22
				},
				"ttft_ms": {
					"n": 6,
					"median": 23.163,
					"mean": 21.336,
					"stddev": 4.146,
					"min": 12.982,
					"max": 23.705,
					"p25": 22.09,
					"p75": 23.243
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 492012696,
					"mean": 493894113.333,
					"stddev": 5404204.457,
					"min": 490407064,
					"max": 504808600,
					"p25": 491529368,
					"p75": 492569698
				},
				"prefill_tps": {
					"n": 6,
					"median": 11475.427,
					"mean": 13220.591,
					"stddev": 4054.634,
					"min": 11175.623,
					"max": 21451.316,
					"p25": 11365.563,
					"p75": 12181.466
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mirai-l, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 776.066,
					"mean": 775.504,
					"stddev": 2.943,
					"min": 771.371,
					"max": 779.552,
					"p25": 773.624,
					"p75": 776.884
				},
				"ttft_ms": {
					"n": 6,
					"median": 88.753,
					"mean": 88.557,
					"stddev": 1.446,
					"min": 86.86,
					"max": 90.287,
					"p25": 87.294,
					"p75": 89.601
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 754845040,
					"mean": 755232790.667,
					"stddev": 1010503.19,
					"min": 754451800,
					"max": 757056880,
					"p25": 754529624,
					"p75": 755541384
				},
				"prefill_tps": {
					"n": 6,
					"median": 66556.979,
					"mean": 67305.493,
					"stddev": 2594.678,
					"min": 65117.166,
					"max": 71324.093,
					"p25": 65167.936,
					"p75": 68818.857
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: litert-int8, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 198.867,
					"mean": 199.22,
					"stddev": 1.014,
					"min": 198.257,
					"max": 200.557,
					"p25": 198.442,
					"p75": 200.071
				},
				"ttft_ms": {
					"n": 6,
					"median": 635.294,
					"mean": 635.771,
					"stddev": 4.637,
					"min": 631.129,
					"max": 642.685,
					"p25": 631.875,
					"p75": 638.44
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1342080824,
					"mean": 1342056232,
					"stddev": 916441.585,
					"min": 1341032224,
					"max": 1343588152,
					"p25": 1341413164,
					"p75": 1342293774
				},
				"prefill_tps": {
					"n": 6,
					"median": 3308.063,
					"mean": 3306.26,
					"stddev": 23.841,
					"min": 3268.412,
					"max": 3330.336,
					"p25": 3295.634,
					"p75": 3325.411
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 747.925,
					"mean": 747.427,
					"stddev": 8.917,
					"min": 731.192,
					"max": 756.75,
					"p25": 746.759,
					"p75": 752.781
				},
				"ttft_ms": {
					"n": 6,
					"median": 44.407,
					"mean": 44.392,
					"stddev": 0.099,
					"min": 44.257,
					"max": 44.489,
					"p25": 44.325,
					"p75": 44.473
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1118930300,
					"mean": 1119348076,
					"stddev": 3566325.733,
					"min": 1115981144,
					"max": 1123124592,
					"p25": 1116181872,
					"p75": 1122624868
				},
				"prefill_tps": {
					"n": 6,
					"median": 46119.478,
					"mean": 46134.629,
					"stddev": 102.554,
					"min": 46033.851,
					"max": 46275.166,
					"p25": 46050.413,
					"p75": 46204.175
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 599.8,
					"mean": 598.369,
					"stddev": 6.474,
					"min": 586.277,
					"max": 603.928,
					"p25": 597.503,
					"p75": 602.705
				},
				"ttft_ms": {
					"n": 6,
					"median": 61.549,
					"mean": 61.558,
					"stddev": 0.065,
					"min": 61.471,
					"max": 61.669,
					"p25": 61.537,
					"p75": 61.567
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 651601096,
					"mean": 651612010.667,
					"stddev": 561607.445,
					"min": 650986696,
					"max": 652281032,
					"p25": 651125942,
					"p75": 652076214
				},
				"prefill_tps": {
					"n": 6,
					"median": 34935.099,
					"mean": 34944.626,
					"stddev": 90.209,
					"min": 34835.264,
					"max": 35107.568,
					"p25": 34908.894,
					"p75": 34949.7
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 712.245,
					"mean": 712.766,
					"stddev": 1.314,
					"min": 711.297,
					"max": 714.643,
					"p25": 712.053,
					"p75": 713.695
				},
				"ttft_ms": {
					"n": 6,
					"median": 61.869,
					"mean": 61.779,
					"stddev": 2.163,
					"min": 59.492,
					"max": 65.262,
					"p25": 60.09,
					"p75": 62.482
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2202879548,
					"mean": 2248200410.667,
					"stddev": 190078180.429,
					"min": 2048394824,
					"max": 2522842672,
					"p25": 2104968758,
					"p75": 2380625420
				},
				"prefill_tps": {
					"n": 6,
					"median": 33457.766,
					"mean": 33526.843,
					"stddev": 1168.62,
					"min": 31692.974,
					"max": 34790.29,
					"p25": 33101.153,
					"p75": 34449.426
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 702.152,
					"mean": 700.98,
					"stddev": 2.798,
					"min": 695.467,
					"max": 703.071,
					"p25": 701.127,
					"p75": 702.227
				},
				"ttft_ms": {
					"n": 6,
					"median": 57.72,
					"mean": 57.636,
					"stddev": 0.269,
					"min": 57.239,
					"max": 57.882,
					"p25": 57.45,
					"p75": 57.85
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2239546544,
					"mean": 2259595102.667,
					"stddev": 175726793.485,
					"min": 2085700808,
					"max": 2556314800,
					"p25": 2123732144,
					"p75": 2323387568
				},
				"prefill_tps": {
					"n": 6,
					"median": 35879.866,
					"mean": 35931.434,
					"stddev": 169.349,
					"min": 35780.426,
					"max": 36186.303,
					"p25": 35792.942,
					"p75": 36043.598
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 592.146,
					"mean": 589.666,
					"stddev": 18.316,
					"min": 568.826,
					"max": 612.134,
					"p25": 573.362,
					"p75": 602.269
				},
				"ttft_ms": {
					"n": 6,
					"median": 63.42,
					"mean": 64.364,
					"stddev": 3.059,
					"min": 61.923,
					"max": 70.317,
					"p25": 62.692,
					"p75": 64.348
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 653583596,
					"mean": 653441593.333,
					"stddev": 865439.852,
					"min": 651871456,
					"max": 654361800,
					"p25": 653296846,
					"p75": 653931786
				},
				"prefill_tps": {
					"n": 6,
					"median": 32937.794,
					"mean": 32480.41,
					"stddev": 1541.314,
					"min": 29504.56,
					"max": 33763.065,
					"p25": 32444.983,
					"p75": 33328.35
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mirai-l, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 794.376,
					"mean": 793.753,
					"stddev": 1.944,
					"min": 790.351,
					"max": 795.455,
					"p25": 793.049,
					"p75": 795.109
				},
				"ttft_ms": {
					"n": 6,
					"median": 69.178,
					"mean": 69.096,
					"stddev": 1.445,
					"min": 66.887,
					"max": 71.375,
					"p25": 68.719,
					"p75": 69.34
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 718046588,
					"mean": 725151786.667,
					"stddev": 17871403.972,
					"min": 717489496,
					"max": 761628064,
					"p25": 717722992,
					"p75": 718075290
				},
				"prefill_tps": {
					"n": 6,
					"median": 19693.066,
					"mean": 19715.351,
					"stddev": 717.106,
					"min": 18754.579,
					"max": 20522.687,
					"p25": 19259.237,
					"p75": 20308.829
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: litert-int8, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 199.985,
					"mean": 199.693,
					"stddev": 1.389,
					"min": 197.678,
					"max": 201.043,
					"p25": 198.743,
					"p75": 200.849
				},
				"ttft_ms": {
					"n": 6,
					"median": 136.239,
					"mean": 136.7,
					"stddev": 1.263,
					"min": 135.746,
					"max": 139.102,
					"p25": 135.927,
					"p75": 136.848
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 836158892,
					"mean": 836063318.667,
					"stddev": 495047.864,
					"min": 835437984,
					"max": 836617656,
					"p25": 835646904,
					"p75": 836437408
				},
				"prefill_tps": {
					"n": 6,
					"median": 2103.127,
					"mean": 2092.999,
					"stddev": 22.231,
					"min": 2050.28,
					"max": 2107.568,
					"p25": 2090.919,
					"p75": 2106.065
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 764.695,
					"mean": 764.388,
					"stddev": 3.276,
					"min": 759.07,
					"max": 768.923,
					"p25": 763.358,
					"p75": 765.699
				},
				"ttft_ms": {
					"n": 6,
					"median": 18.043,
					"mean": 18.335,
					"stddev": 0.558,
					"min": 17.841,
					"max": 19.052,
					"p25": 17.99,
					"p75": 18.804
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1115063676,
					"mean": 1114856129.333,
					"stddev": 2520150.674,
					"min": 1110738264,
					"max": 1117472088,
					"p25": 1113814390,
					"p75": 1116816752
				},
				"prefill_tps": {
					"n": 6,
					"median": 14188.388,
					"mean": 13973.23,
					"stddev": 418.073,
					"min": 13436.91,
					"max": 14348.971,
					"p25": 13621.209,
					"p75": 14230.527
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 630.326,
					"mean": 631.31,
					"stddev": 7.168,
					"min": 624.643,
					"max": 645.106,
					"p25": 627.727,
					"p75": 630.53
				},
				"ttft_ms": {
					"n": 6,
					"median": 21.365,
					"mean": 21.31,
					"stddev": 0.146,
					"min": 21.114,
					"max": 21.469,
					"p25": 21.195,
					"p75": 21.396
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 641189028,
					"mean": 642969438.667,
					"stddev": 4302607.878,
					"min": 640894104,
					"max": 651724000,
					"p25": 641004714,
					"p75": 641705154
				},
				"prefill_tps": {
					"n": 6,
					"median": 12906.516,
					"mean": 12920.713,
					"stddev": 109.887,
					"min": 12819.229,
					"max": 13117.442,
					"p25": 12839.855,
					"p75": 12944.336
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 799.464,
					"mean": 802.318,
					"stddev": 10.146,
					"min": 794.291,
					"max": 820.027,
					"p25": 794.481,
					"p75": 805.746
				},
				"ttft_ms": {
					"n": 6,
					"median": 29.246,
					"mean": 28.43,
					"stddev": 2.678,
					"min": 25.146,
					"max": 31.271,
					"p25": 26.029,
					"p75": 30.349
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 965289604,
					"mean": 968623772,
					"stddev": 15394781.952,
					"min": 948389568,
					"max": 988645056,
					"p25": 960636506,
					"p75": 980104896
				},
				"prefill_tps": {
					"n": 6,
					"median": 8948.558,
					"mean": 9259.169,
					"stddev": 908.451,
					"min": 8338.491,
					"max": 10390.032,
					"p25": 8602.255,
					"p75": 10069.055
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 818.526,
					"mean": 818.343,
					"stddev": 1.976,
					"min": 814.661,
					"max": 820.477,
					"p25": 818.389,
					"p75": 819.277
				},
				"ttft_ms": {
					"n": 6,
					"median": 23.591,
					"mean": 23.621,
					"stddev": 0.182,
					"min": 23.435,
					"max": 23.852,
					"p25": 23.467,
					"p75": 23.772
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 724084020,
					"mean": 717953689.333,
					"stddev": 11363240.598,
					"min": 696845632,
					"max": 724944216,
					"p25": 715453736,
					"p75": 724911460
				},
				"prefill_tps": {
					"n": 6,
					"median": 11081.556,
					"mean": 11068.68,
					"stddev": 86.445,
					"min": 10953.277,
					"max": 11167.823,
					"p25": 11003.06,
					"p75": 11133.619
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16, magnitude default, model-dtype, uzu default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 617.192,
					"mean": 621.635,
					"stddev": 26.514,
					"min": 590.165,
					"max": 655.207,
					"p25": 602.721,
					"p75": 643.416
				},
				"ttft_ms": {
					"n": 6,
					"median": 22.577,
					"mean": 22.511,
					"stddev": 0.45,
					"min": 21.778,
					"max": 23.008,
					"p25": 22.311,
					"p75": 22.823
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 641877216,
					"mean": 643843304,
					"stddev": 5196284.645,
					"min": 641189088,
					"max": 654427408,
					"p25": 641598688,
					"p75": 642106592
				},
				"prefill_tps": {
					"n": 6,
					"median": 11912.796,
					"mean": 11912.63,
					"stddev": 273.597,
					"min": 11629.492,
					"max": 12349.848,
					"p25": 11686.301,
					"p75": 12023.231
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 150.851,
					"mean": 150.003,
					"stddev": 2.398,
					"min": 145.183,
					"max": 151.562,
					"p25": 150.453,
					"p75": 151.15
				},
				"ttft_ms": {
					"n": 6,
					"median": 129.845,
					"mean": 129.749,
					"stddev": 0.549,
					"min": 128.796,
					"max": 130.467,
					"p25": 129.649,
					"p75": 129.93
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5099966664,
					"mean": 5129397773.333,
					"stddev": 90698988.393,
					"min": 5067870336,
					"max": 5309239520,
					"p25": 5078347928,
					"p75": 5121142996
				},
				"prefill_tps": {
					"n": 6,
					"median": 15772.655,
					"mean": 15784.517,
					"stddev": 66.941,
					"min": 15697.456,
					"max": 15901.115,
					"p25": 15762.304,
					"p75": 15796.442
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 199.61,
					"mean": 199.41,
					"stddev": 1.242,
					"min": 197.515,
					"max": 200.839,
					"p25": 198.699,
					"p75": 200.269
				},
				"ttft_ms": {
					"n": 6,
					"median": 113.412,
					"mean": 114.261,
					"stddev": 2.26,
					"min": 113.066,
					"max": 118.843,
					"p25": 113.146,
					"p75": 113.685
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3177563564,
					"mean": 3177025646.667,
					"stddev": 3441295.259,
					"min": 3172959744,
					"max": 3181381048,
					"p25": 3174065634,
					"p75": 3179230618
				},
				"prefill_tps": {
					"n": 6,
					"median": 18058.058,
					"mean": 17929.542,
					"stddev": 343.812,
					"min": 17232.82,
					"max": 18113.314,
					"p25": 18014.712,
					"p75": 18100.569
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 197.574,
					"mean": 196.563,
					"stddev": 2.676,
					"min": 191.327,
					"max": 198.447,
					"p25": 196.575,
					"p75": 198.054
				},
				"ttft_ms": {
					"n": 6,
					"median": 135.981,
					"mean": 136.014,
					"stddev": 1.046,
					"min": 134.603,
					"max": 137.682,
					"p25": 135.495,
					"p75": 136.372
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2628479568,
					"mean": 2628520524,
					"stddev": 472021.128,
					"min": 2628012624,
					"max": 2629405264,
					"p25": 2628311614,
					"p75": 2628487760
				},
				"prefill_tps": {
					"n": 6,
					"median": 15384.909,
					"mean": 15380.184,
					"stddev": 114.269,
					"min": 15184.205,
					"max": 15533.391,
					"p25": 15364.363,
					"p75": 15423.361
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 195.075,
					"mean": 194.933,
					"stddev": 1.886,
					"min": 192.362,
					"max": 196.953,
					"p25": 193.588,
					"p75": 196.55
				},
				"ttft_ms": {
					"n": 6,
					"median": 100.339,
					"mean": 100.029,
					"stddev": 2.138,
					"min": 96.345,
					"max": 102.215,
					"p25": 99.39,
					"p75": 101.482
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3707759532,
					"mean": 3701044806.667,
					"stddev": 17280391.404,
					"min": 3667446640,
					"max": 3713305480,
					"p25": 3700317088,
					"p75": 3711060920
				},
				"prefill_tps": {
					"n": 6,
					"median": 20564.107,
					"mean": 20639.799,
					"stddev": 451.255,
					"min": 20183.902,
					"max": 21430.44,
					"p25": 20343.432,
					"p75": 20760.799
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 198.346,
					"mean": 198.426,
					"stddev": 0.511,
					"min": 197.805,
					"max": 199.153,
					"p25": 198.088,
					"p75": 198.764
				},
				"ttft_ms": {
					"n": 6,
					"median": 94.232,
					"mean": 94.349,
					"stddev": 0.461,
					"min": 93.859,
					"max": 95.044,
					"p25": 94.019,
					"p75": 94.643
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3495692784,
					"mean": 3493259776,
					"stddev": 4481260.46,
					"min": 3487140360,
					"max": 3496839712,
					"p25": 3489823234,
					"p75": 3496167920
				},
				"prefill_tps": {
					"n": 6,
					"median": 21871.608,
					"mean": 21843.702,
					"stddev": 105.609,
					"min": 21688.942,
					"max": 21956.345,
					"p25": 21771.744,
					"p75": 21919.341
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 184.215,
					"mean": 183.381,
					"stddev": 5.777,
					"min": 173.032,
					"max": 190.556,
					"p25": 182.873,
					"p75": 185.435
				},
				"ttft_ms": {
					"n": 6,
					"median": 141.907,
					"mean": 141.953,
					"stddev": 0.956,
					"min": 140.652,
					"max": 143.527,
					"p25": 141.551,
					"p75": 142.197
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2630855248,
					"mean": 2630759690.667,
					"stddev": 462798.403,
					"min": 2629880424,
					"max": 2631174808,
					"p25": 2630740542,
					"p75": 2631031394
				},
				"prefill_tps": {
					"n": 6,
					"median": 14523.121,
					"mean": 14520.396,
					"stddev": 99.355,
					"min": 14360.039,
					"max": 14659.356,
					"p25": 14492.549,
					"p75": 14561.565
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 192.605,
					"mean": 185.125,
					"stddev": 18.374,
					"min": 147.648,
					"max": 193.261,
					"p25": 191.641,
					"p75": 193.136
				},
				"ttft_ms": {
					"n": 6,
					"median": 24.93,
					"mean": 25.165,
					"stddev": 0.67,
					"min": 24.768,
					"max": 26.504,
					"p25": 24.788,
					"p75": 25.069
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5290406016,
					"mean": 5250382678.667,
					"stddev": 121687319.122,
					"min": 5070475416,
					"max": 5360832736,
					"p25": 5166149832,
					"p75": 5346685092
				},
				"prefill_tps": {
					"n": 6,
					"median": 10269.142,
					"mean": 10178.84,
					"stddev": 260.586,
					"min": 9658.919,
					"max": 10335.917,
					"p25": 10211.923,
					"p75": 10327.585
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 199.559,
					"mean": 198.965,
					"stddev": 2.083,
					"min": 194.789,
					"max": 200.387,
					"p25": 199.389,
					"p75": 200.011
				},
				"ttft_ms": {
					"n": 6,
					"median": 28.682,
					"mean": 27.528,
					"stddev": 4.008,
					"min": 19.578,
					"max": 30.974,
					"p25": 28.423,
					"p75": 28.858
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 3169928656,
					"mean": 3168738057.333,
					"stddev": 5397365.514,
					"min": 3160393072,
					"max": 3175040464,
					"p25": 3165681074,
					"p75": 3172136376
				},
				"prefill_tps": {
					"n": 6,
					"median": 8925.636,
					"mean": 9513.225,
					"stddev": 1766.659,
					"min": 8264.996,
					"max": 13075.902,
					"p25": 8871.086,
					"p75": 9007.119
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 199.635,
					"mean": 199.774,
					"stddev": 0.376,
					"min": 199.414,
					"max": 200.348,
					"p25": 199.509,
					"p75": 200.018
				},
				"ttft_ms": {
					"n": 6,
					"median": 33.804,
					"mean": 33.921,
					"stddev": 0.856,
					"min": 33.021,
					"max": 35.078,
					"p25": 33.201,
					"p75": 34.565
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2611194424,
					"mean": 2614072554.667,
					"stddev": 7083832.968,
					"min": 2610383416,
					"max": 2628487784,
					"p25": 2611071526,
					"p75": 2611907146
				},
				"prefill_tps": {
					"n": 6,
					"median": 7899.791,
					"mean": 7872.598,
					"stddev": 198.364,
					"min": 7605.24,
					"max": 8085.147,
					"p25": 7723.776,
					"p75": 8035.332
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 198.027,
					"mean": 197.204,
					"stddev": 1.76,
					"min": 194.616,
					"max": 199.029,
					"p25": 196.046,
					"p75": 198.112
				},
				"ttft_ms": {
					"n": 6,
					"median": 32.718,
					"mean": 32.603,
					"stddev": 1.235,
					"min": 31.124,
					"max": 34.018,
					"p25": 31.554,
					"p75": 33.586
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2900093904,
					"mean": 2894009978.667,
					"stddev": 15608667.772,
					"min": 2862189544,
					"max": 2901167056,
					"p25": 2899405776,
					"p75": 2901027774
				},
				"prefill_tps": {
					"n": 6,
					"median": 7969.013,
					"mean": 8006.111,
					"stddev": 310.633,
					"min": 7656.189,
					"max": 8386.568,
					"p25": 7760.226,
					"p75": 8266.192
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 198.968,
					"mean": 199.186,
					"stddev": 1.659,
					"min": 197.354,
					"max": 202.172,
					"p25": 198.266,
					"p75": 199.459
				},
				"ttft_ms": {
					"n": 6,
					"median": 30.656,
					"mean": 30.607,
					"stddev": 0.116,
					"min": 30.449,
					"max": 30.734,
					"p25": 30.521,
					"p75": 30.666
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2621393464,
					"mean": 2619378236,
					"stddev": 12288153.108,
					"min": 2598734416,
					"max": 2630552120,
					"p25": 2614962750,
					"p75": 2628880946
				},
				"prefill_tps": {
					"n": 6,
					"median": 8473.034,
					"mean": 8487.516,
					"stddev": 33.419,
					"min": 8453.309,
					"max": 8530.49,
					"p25": 8467.919,
					"p75": 8515.022
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 190.541,
					"mean": 188.867,
					"stddev": 6.035,
					"min": 177.943,
					"max": 193.984,
					"p25": 187.316,
					"p75": 193.101
				},
				"ttft_ms": {
					"n": 6,
					"median": 36.667,
					"mean": 36.951,
					"stddev": 1.113,
					"min": 35.813,
					"max": 38.456,
					"p25": 36.095,
					"p75": 37.816
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 2613848644,
					"mean": 2616751362.667,
					"stddev": 7516314.475,
					"min": 2613070392,
					"max": 2632075904,
					"p25": 2613594698,
					"p75": 2614078068
				},
				"prefill_tps": {
					"n": 6,
					"median": 7122.414,
					"mean": 7070.545,
					"stddev": 219.949,
					"min": 6778.584,
					"max": 7297.189,
					"p25": 6897.057,
					"p75": 7241.151
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-zml-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 261.009,
					"mean": 261.204,
					"stddev": 1.706,
					"min": 258.984,
					"max": 263.266,
					"p25": 260.101,
					"p75": 262.622
				},
				"ttft_ms": {
					"n": 6,
					"median": 59.636,
					"mean": 59.661,
					"stddev": 0.354,
					"min": 59.233,
					"max": 60.156,
					"p25": 59.402,
					"p75": 59.895
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5231849072,
					"mean": 5217794314.667,
					"stddev": 67165362.929,
					"min": 5085499040,
					"max": 5270195752,
					"p25": 5227847232,
					"p75": 5253607018
				},
				"prefill_tps": {
					"n": 6,
					"median": 34341.714,
					"mean": 34328.288,
					"stddev": 203.386,
					"min": 34044.817,
					"max": 34575.321,
					"p25": 34193.576,
					"p75": 34476.901
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 510.079,
					"mean": 507.31,
					"stddev": 11.418,
					"min": 491.612,
					"max": 519.035,
					"p25": 498.699,
					"p75": 516.13
				},
				"ttft_ms": {
					"n": 6,
					"median": 48.897,
					"mean": 48.879,
					"stddev": 0.138,
					"min": 48.729,
					"max": 49.015,
					"p25": 48.75,
					"p75": 49
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1445316012,
					"mean": 1445946820,
					"stddev": 2192149.564,
					"min": 1443612088,
					"max": 1449100728,
					"p25": 1444468182,
					"p75": 1447441884
				},
				"prefill_tps": {
					"n": 6,
					"median": 41884.517,
					"mean": 41899.804,
					"stddev": 117.918,
					"min": 41783.128,
					"max": 42028.361,
					"p25": 41795.92,
					"p75": 42010.063
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 468.972,
					"mean": 463.368,
					"stddev": 17.064,
					"min": 429.009,
					"max": 473.748,
					"p25": 466.883,
					"p75": 472.234
				},
				"ttft_ms": {
					"n": 6,
					"median": 63.4,
					"mean": 63.384,
					"stddev": 0.131,
					"min": 63.218,
					"max": 63.515,
					"p25": 63.283,
					"p75": 63.496
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 983344616,
					"mean": 983560306.667,
					"stddev": 1128140.209,
					"min": 982304184,
					"max": 985531880,
					"p25": 982914488,
					"p75": 983885228
				},
				"prefill_tps": {
					"n": 6,
					"median": 33834.468,
					"mean": 33821.442,
					"stddev": 66.84,
					"min": 33703.058,
					"max": 33880.36,
					"p25": 33802.076,
					"p75": 33872.381
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 532.227,
					"mean": 531.867,
					"stddev": 3.271,
					"min": 526.873,
					"max": 535.562,
					"p25": 530.049,
					"p75": 534.301
				},
				"ttft_ms": {
					"n": 6,
					"median": 46.921,
					"mean": 47.24,
					"stddev": 1.209,
					"min": 45.86,
					"max": 49.273,
					"p25": 46.592,
					"p75": 47.715
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1377838880,
					"mean": 1387393478.667,
					"stddev": 26065294.23,
					"min": 1362003720,
					"max": 1432127240,
					"p25": 1371588402,
					"p75": 1398245152
				},
				"prefill_tps": {
					"n": 6,
					"median": 44253.527,
					"mean": 43973.772,
					"stddev": 1113.236,
					"min": 42124.316,
					"max": 45257.668,
					"p25": 43504.844,
					"p75": 44587.116
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 526.898,
					"mean": 527.138,
					"stddev": 1.118,
					"min": 525.998,
					"max": 529.094,
					"p25": 526.441,
					"p75": 527.462
				},
				"ttft_ms": {
					"n": 6,
					"median": 38.719,
					"mean": 37.329,
					"stddev": 3.417,
					"min": 30.355,
					"max": 38.793,
					"p25": 38.636,
					"p75": 38.767
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1087079840,
					"mean": 1081473809.333,
					"stddev": 11810651.308,
					"min": 1059161432,
					"max": 1089668512,
					"p25": 1079481940,
					"p75": 1088447904
				},
				"prefill_tps": {
					"n": 6,
					"median": 53684.954,
					"mean": 56183.828,
					"stddev": 6154.768,
					"min": 53567.692,
					"max": 68745.594,
					"p25": 53601.01,
					"p75": 53806.296
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 442.677,
					"mean": 443.556,
					"stddev": 9.999,
					"min": 427.312,
					"max": 456.744,
					"p25": 441.244,
					"p75": 449.041
				},
				"ttft_ms": {
					"n": 6,
					"median": 66.199,
					"mean": 66.3,
					"stddev": 2.131,
					"min": 63.947,
					"max": 69.268,
					"p25": 64.516,
					"p75": 67.726
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 985810372,
					"mean": 985843164,
					"stddev": 1041129.081,
					"min": 984647144,
					"max": 987268584,
					"p25": 985064924,
					"p75": 986482146
				},
				"prefill_tps": {
					"n": 6,
					"median": 31445.787,
					"mean": 31433.085,
					"stddev": 1104.764,
					"min": 29964.3,
					"max": 32682.763,
					"p25": 30650.617,
					"p75": 32367.182
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-zml-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 420.171,
					"mean": 419.936,
					"stddev": 1.832,
					"min": 417.338,
					"max": 422.455,
					"p25": 418.816,
					"p75": 420.882
				},
				"ttft_ms": {
					"n": 6,
					"median": 15.916,
					"mean": 15.695,
					"stddev": 0.5,
					"min": 14.868,
					"max": 16.152,
					"p25": 15.445,
					"p75": 16
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5245316708,
					"mean": 5252173428,
					"stddev": 32318282.02,
					"min": 5210836616,
					"max": 5303455368,
					"p25": 5236580000,
					"p75": 5267164730
				},
				"prefill_tps": {
					"n": 6,
					"median": 16084.561,
					"mean": 16325.598,
					"stddev": 533.194,
					"min": 15849.43,
					"max": 17218.187,
					"p25": 16000.534,
					"p75": 16579.383
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default.",
				"Its phases were timed by the harness clock around the engine API, so they include transport that engine counters leave out."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 524.065,
					"mean": 522.154,
					"stddev": 5.19,
					"min": 512.142,
					"max": 525.967,
					"p25": 521.825,
					"p75": 525.22
				},
				"ttft_ms": {
					"n": 6,
					"median": 19.055,
					"mean": 19.126,
					"stddev": 0.215,
					"min": 18.897,
					"max": 19.487,
					"p25": 19.005,
					"p75": 19.221
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1444505016,
					"mean": 1442514348,
					"stddev": 5816701.784,
					"min": 1431897480,
					"max": 1447265696,
					"p25": 1440994762,
					"p75": 1446442406
				},
				"prefill_tps": {
					"n": 6,
					"median": 13434.834,
					"mean": 13386.083,
					"stddev": 149.119,
					"min": 13136.963,
					"max": 13547.124,
					"p25": 13319.152,
					"p75": 13470.32
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 479.469,
					"mean": 479.467,
					"stddev": 2.697,
					"min": 475.447,
					"max": 483.209,
					"p25": 478.25,
					"p75": 480.891
				},
				"ttft_ms": {
					"n": 6,
					"median": 22.258,
					"mean": 22.272,
					"stddev": 0.134,
					"min": 22.07,
					"max": 22.433,
					"p25": 22.217,
					"p75": 22.373
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 973735352,
					"mean": 975657733.333,
					"stddev": 4121713.595,
					"min": 972604856,
					"max": 983500168,
					"p25": 973452746,
					"p75": 976192934
				},
				"prefill_tps": {
					"n": 6,
					"median": 12319.343,
					"mean": 12308.067,
					"stddev": 74.254,
					"min": 12186.414,
					"max": 12385.099,
					"p25": 12277.443,
					"p75": 12360.881
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 559.043,
					"mean": 558.416,
					"stddev": 2.26,
					"min": 554.196,
					"max": 560.261,
					"p25": 558.038,
					"p75": 559.946
				},
				"ttft_ms": {
					"n": 6,
					"median": 26.212,
					"mean": 25.311,
					"stddev": 2.64,
					"min": 20.18,
					"max": 27.36,
					"p25": 25.26,
					"p75": 26.771
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1252083500,
					"mean": 1248763033.333,
					"stddev": 20148648.597,
					"min": 1224820536,
					"max": 1269860176,
					"p25": 1231042366,
					"p75": 1265297250
				},
				"prefill_tps": {
					"n": 6,
					"median": 9995.124,
					"mean": 10468.706,
					"stddev": 1311.972,
					"min": 9553.665,
					"max": 13066.558,
					"p25": 9771.982,
					"p75": 10376.906
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 571.702,
					"mean": 570.297,
					"stddev": 3.274,
					"min": 565.09,
					"max": 573.164,
					"p25": 568.386,
					"p75": 572.558
				},
				"ttft_ms": {
					"n": 6,
					"median": 18.133,
					"mean": 16.719,
					"stddev": 3.631,
					"min": 9.312,
					"max": 18.348,
					"p25": 18.065,
					"p75": 18.292
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 1011516844,
					"mean": 1004804865.333,
					"stddev": 12142578.123,
					"min": 981992864,
					"max": 1012090296,
					"p25": 1002735014,
					"p75": 1011807666
				},
				"prefill_tps": {
					"n": 6,
					"median": 14489.526,
					"mean": 16828.182,
					"stddev": 5891.021,
					"min": 14307.271,
					"max": 28851.572,
					"p25": 14347.663,
					"p75": 14520.498
				}
			}
		},
		{
			"experiment_id": "s3-lfm2.5-350m-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, magnitude default, model-dtype, zml default."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 464.211,
					"mean": 462.094,
					"stddev": 15.635,
					"min": 437.928,
					"max": 478.77,
					"p25": 453.757,
					"p75": 473.93
				},
				"ttft_ms": {
					"n": 6,
					"median": 23.379,
					"mean": 22.162,
					"stddev": 4.092,
					"min": 14.155,
					"max": 25.968,
					"p25": 22.705,
					"p75": 23.553
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 974775784,
					"mean": 976485181.333,
					"stddev": 4757832.068,
					"min": 973653504,
					"max": 986105296,
					"p25": 974210554,
					"p75": 975377878
				},
				"prefill_tps": {
					"n": 6,
					"median": 11298.959,
					"mean": 12487.304,
					"stddev": 3269.466,
					"min": 10149.869,
					"max": 19057.545,
					"p25": 11215.661,
					"p75": 11772.69
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 92.83,
					"mean": 93.037,
					"stddev": 3.137,
					"min": 88.398,
					"max": 97.066,
					"p25": 91.529,
					"p75": 95.208
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 527.516,
					"mean": 527.925,
					"stddev": 1.543,
					"min": 526.128,
					"max": 530.421,
					"p25": 527.075,
					"p75": 528.658
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 133.267,
					"mean": 132.852,
					"stddev": 1.485,
					"min": 130.833,
					"max": 134.258,
					"p25": 131.679,
					"p75": 134.068
				},
				"ttft_ms": {
					"n": 6,
					"median": 2430.924,
					"mean": 2436.151,
					"stddev": 36.813,
					"min": 2385.487,
					"max": 2499.096,
					"p25": 2426.781,
					"p75": 2441.537
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 845.458,
					"mean": 843.674,
					"stddev": 12.506,
					"min": 822.913,
					"max": 861.777,
					"p25": 841.602,
					"p75": 845.955
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 92.178,
					"mean": 92.232,
					"stddev": 0.232,
					"min": 91.995,
					"max": 92.66,
					"p25": 92.109,
					"p75": 92.265
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 441.765,
					"mean": 441.62,
					"stddev": 3.907,
					"min": 435.267,
					"max": 446.212,
					"p25": 440.224,
					"p75": 444.193
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 167.541,
					"mean": 167.196,
					"stddev": 5.809,
					"min": 159.135,
					"max": 173.994,
					"p25": 163.318,
					"p75": 171.674
				},
				"ttft_ms": {
					"n": 6,
					"median": 208.895,
					"mean": 209.069,
					"stddev": 4.484,
					"min": 202.897,
					"max": 216.773,
					"p25": 207.745,
					"p75": 209.421
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 1263.294,
					"mean": 1262.783,
					"stddev": 25.612,
					"min": 1221.491,
					"max": 1301.349,
					"p25": 1258.418,
					"p75": 1268.68
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 76.538,
					"mean": 76.081,
					"stddev": 0.885,
					"min": 74.405,
					"max": 76.666,
					"p25": 75.951,
					"p75": 76.571
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 851.598,
					"mean": 850.535,
					"stddev": 6.836,
					"min": 841.926,
					"max": 859.703,
					"p25": 845.141,
					"p75": 854.446
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 100.144,
					"mean": 100.262,
					"stddev": 0.862,
					"min": 99.089,
					"max": 101.668,
					"p25": 99.908,
					"p75": 100.557
				},
				"ttft_ms": {
					"n": 6,
					"median": 1647.625,
					"mean": 1642.977,
					"stddev": 29.903,
					"min": 1587.013,
					"max": 1672.761,
					"p25": 1641.354,
					"p75": 1659.585
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 1252.251,
					"mean": 1253.922,
					"stddev": 24.172,
					"min": 1229.544,
					"max": 1298.087,
					"p25": 1238.817,
					"p75": 1255.858
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 77,
					"mean": 76.147,
					"stddev": 2.102,
					"min": 71.94,
					"max": 77.363,
					"p25": 76.408,
					"p75": 77.278
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 850.823,
					"mean": 815.963,
					"stddev": 129.024,
					"min": 556.4,
					"max": 898.627,
					"p25": 849.326,
					"p75": 880.412
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 121.148,
					"mean": 121.615,
					"stddev": 1.902,
					"min": 119.584,
					"max": 124.079,
					"p25": 120.184,
					"p75": 123.189
				},
				"ttft_ms": {
					"n": 6,
					"median": 150.815,
					"mean": 163.072,
					"stddev": 31.91,
					"min": 139.321,
					"max": 225.315,
					"p25": 146.265,
					"p75": 163.266
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 1771.535,
					"mean": 1687.162,
					"stddev": 265.219,
					"min": 1203.024,
					"max": 1940.32,
					"p25": 1628.389,
					"p75": 1834.795
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 14.343,
					"mean": 14.32,
					"stddev": 0.091,
					"min": 14.145,
					"max": 14.394,
					"p25": 14.319,
					"p75": 14.374
				},
				"ttft_ms": {
					"n": 6,
					"median": 93993.255,
					"mean": 93808.552,
					"stddev": 505.936,
					"min": 92780.753,
					"max": 94093.475,
					"p25": 93958.604,
					"p75": 94030.954
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 21.79,
					"mean": 21.834,
					"stddev": 0.118,
					"min": 21.768,
					"max": 22.075,
					"p25": 21.783,
					"p75": 21.799
				}
			}
		},
		{
			"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-oneplus-13--android-16--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 17.425,
					"mean": 17.446,
					"stddev": 0.134,
					"min": 17.309,
					"max": 17.637,
					"p25": 17.334,
					"p75": 17.538
				},
				"ttft_ms": {
					"n": 6,
					"median": 10768.615,
					"mean": 10754.461,
					"stddev": 55.206,
					"min": 10677.615,
					"max": 10814.436,
					"p25": 10713.187,
					"p75": 10794.235
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 23.789,
					"mean": 23.819,
					"stddev": 0.123,
					"min": 23.686,
					"max": 23.99,
					"p25": 23.731,
					"p75": 23.911
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 45.41,
					"mean": 45.341,
					"stddev": 2.042,
					"min": 42.962,
					"max": 48.05,
					"p25": 43.659,
					"p75": 46.704
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 311.929,
					"mean": 306.422,
					"stddev": 14.877,
					"min": 276.247,
					"max": 314.48,
					"p25": 310.401,
					"p75": 313.524
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 5 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 5,
			"metrics": {
				"decode_tps": {
					"n": 5,
					"median": 39.452,
					"mean": 39.176,
					"stddev": 1.764,
					"min": 36.695,
					"max": 41.573,
					"p25": 38.609,
					"p75": 39.552
				},
				"ttft_ms": {
					"n": 5,
					"median": 12265.994,
					"mean": 12358.909,
					"stddev": 226.415,
					"min": 12181.67,
					"max": 12717.141,
					"p25": 12186.811,
					"p75": 12442.931
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 5,
					"median": 167.306,
					"mean": 166.094,
					"stddev": 3.006,
					"min": 161.374,
					"max": 168.473,
					"p25": 164.912,
					"p75": 168.402
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 51.922,
					"mean": 51.088,
					"stddev": 2.488,
					"min": 46.174,
					"max": 52.678,
					"p25": 51.252,
					"p75": 52.585
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 248.013,
					"mean": 236.033,
					"stddev": 27.051,
					"min": 200.872,
					"max": 264.201,
					"p25": 213.943,
					"p75": 251.388
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: fp32, litert-lm default, f16.",
				"The experiments here use different 4-bit schemes: 8da4w, litert-int4, Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 64.957,
					"mean": 62.627,
					"stddev": 9.591,
					"min": 47.029,
					"max": 73.857,
					"p25": 58.139,
					"p75": 68.06
				},
				"ttft_ms": {
					"n": 6,
					"median": 971.715,
					"mean": 986.714,
					"stddev": 68.745,
					"min": 926.811,
					"max": 1119.153,
					"p25": 948.656,
					"p75": 985.37
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 268.766,
					"mean": 265.486,
					"stddev": 17.406,
					"min": 232.424,
					"max": 281.26,
					"p25": 264.946,
					"p75": 275.712
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 38.97,
					"mean": 38.456,
					"stddev": 1.566,
					"min": 35.286,
					"max": 39.445,
					"p25": 38.895,
					"p75": 39.137
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 454.81,
					"mean": 453.47,
					"stddev": 7.849,
					"min": 438.916,
					"max": 461.674,
					"p25": 452.787,
					"p75": 457.575
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 31.452,
					"mean": 32.246,
					"stddev": 1.728,
					"min": 30.553,
					"max": 34.493,
					"p25": 31.184,
					"p75": 33.685
				},
				"ttft_ms": {
					"n": 6,
					"median": 10590.862,
					"mean": 10671.313,
					"stddev": 350.449,
					"min": 10154.517,
					"max": 11138.349,
					"p25": 10566.913,
					"p75": 10893.482
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 193.841,
					"mean": 192.556,
					"stddev": 6.35,
					"min": 184.301,
					"max": 202.202,
					"p25": 188.513,
					"p75": 194.269
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 41.634,
					"mean": 41.033,
					"stddev": 2.149,
					"min": 37.777,
					"max": 43.246,
					"p25": 39.704,
					"p75": 42.541
				},
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 322.326,
					"mean": 338.231,
					"stddev": 41.151,
					"min": 304.345,
					"max": 414.168,
					"p25": 312.897,
					"p75": 347.93
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: litert-lm default, f16.",
				"The experiments here use different 8-bit schemes: litert-int8, Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 44.808,
					"mean": 44.825,
					"stddev": 2.581,
					"min": 41.458,
					"max": 48.273,
					"p25": 43.053,
					"p75": 46.555
				},
				"ttft_ms": {
					"n": 6,
					"median": 793.636,
					"mean": 987.214,
					"stddev": 480.342,
					"min": 762.111,
					"max": 1965.991,
					"p25": 771.797,
					"p75": 830.954
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 330.695,
					"mean": 298.516,
					"stddev": 82.787,
					"min": 131.318,
					"max": 345.126,
					"p25": 315.295,
					"p75": 339.997
				}
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "failed",
				"detail": "No trial was valid."
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-pixel-8--android-16--lfm2.5-350m--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 5.598,
					"mean": 5.481,
					"stddev": 0.323,
					"min": 4.835,
					"max": 5.712,
					"p25": 5.532,
					"p75": 5.623
				},
				"ttft_ms": {
					"n": 6,
					"median": 16802.87,
					"mean": 16319.033,
					"stddev": 1175.042,
					"min": 13929.993,
					"max": 16902.573,
					"p25": 16647.606,
					"p75": 16860.746
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 15.257,
					"mean": 15.786,
					"stddev": 1.286,
					"min": 15.163,
					"max": 18.403,
					"p25": 15.204,
					"p75": 15.401
				}
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 91.072,
					"mean": 90.892,
					"stddev": 0.672,
					"min": 89.971,
					"max": 91.518,
					"p25": 90.368,
					"p75": 91.455
				},
				"ttft_ms": {
					"n": 6,
					"median": 289.149,
					"mean": 288.961,
					"stddev": 1.02,
					"min": 287.083,
					"max": 289.825,
					"p25": 288.798,
					"p75": 289.696
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5723001656,
					"mean": 5722960692,
					"stddev": 168285.997,
					"min": 5722641208,
					"max": 5723116344,
					"p25": 5722952486,
					"p75": 5723050808
				},
				"prefill_tps": {
					"n": 6,
					"median": 7146.502,
					"mean": 7150.859,
					"stddev": 24.749,
					"min": 7126.676,
					"max": 7195.559,
					"p25": 7134.703,
					"p75": 7155.986
				}
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 88.708,
					"mean": 88.043,
					"stddev": 1.239,
					"min": 85.968,
					"max": 88.915,
					"p25": 87.443,
					"p75": 88.88
				},
				"ttft_ms": {
					"n": 6,
					"median": 305.29,
					"mean": 305.185,
					"stddev": 0.836,
					"min": 303.815,
					"max": 306.408,
					"p25": 305.01,
					"p75": 305.366
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5724762936,
					"mean": 5724782038.667,
					"stddev": 184695.701,
					"min": 5724574472,
					"max": 5725098808,
					"p25": 5724664632,
					"p75": 5724836646
				},
				"prefill_tps": {
					"n": 6,
					"median": 6727.702,
					"mean": 6730.682,
					"stddev": 18.622,
					"min": 6703.94,
					"max": 6761.486,
					"p25": 6726.735,
					"p75": 6734.562
				}
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 91.666,
					"mean": 91.703,
					"stddev": 0.297,
					"min": 91.355,
					"max": 92.171,
					"p25": 91.502,
					"p75": 91.851
				},
				"ttft_ms": {
					"n": 6,
					"median": 57.005,
					"mean": 56.915,
					"stddev": 0.278,
					"min": 56.363,
					"max": 57.129,
					"p25": 56.956,
					"p75": 57.04
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5705577260,
					"mean": 5708403500,
					"stddev": 7016915.36,
					"min": 5705306936,
					"max": 5722723104,
					"p25": 5705478944,
					"p75": 5705737016
				},
				"prefill_tps": {
					"n": 6,
					"median": 4596.134,
					"mean": 4601.772,
					"stddev": 18.564,
					"min": 4585.677,
					"max": 4638.185,
					"p25": 4593.762,
					"p75": 4600.18
				}
			}
		},
		{
			"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 88.068,
					"mean": 87.984,
					"stddev": 0.536,
					"min": 87.136,
					"max": 88.555,
					"p25": 87.719,
					"p75": 88.373
				},
				"ttft_ms": {
					"n": 6,
					"median": 68.825,
					"mean": 68.923,
					"stddev": 0.308,
					"min": 68.685,
					"max": 69.523,
					"p25": 68.746,
					"p75": 68.925
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 5706330924,
					"mean": 5709312804,
					"stddev": 7421034.947,
					"min": 5706142496,
					"max": 5724459856,
					"p25": 5706248974,
					"p75": 5706375956
				},
				"prefill_tps": {
					"n": 6,
					"median": 3759.069,
					"mean": 3754.234,
					"stddev": 16.892,
					"min": 3721.038,
					"max": 3766.811,
					"p25": 3755.318,
					"p75": 3763.779
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.883,
					"mean": 18.848,
					"stddev": 0.076,
					"min": 18.718,
					"max": 18.912,
					"p25": 18.815,
					"p75": 18.896
				},
				"ttft_ms": {
					"n": 6,
					"median": 2747.823,
					"mean": 2817.763,
					"stddev": 210.445,
					"min": 2662.764,
					"max": 3236.837,
					"p25": 2718.037,
					"p75": 2789.375
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 29663270440,
					"mean": 29666331517.333,
					"stddev": 7905708.809,
					"min": 29661369920,
					"max": 29681866232,
					"p25": 29661591098,
					"p75": 29666203176
				},
				"prefill_tps": {
					"n": 6,
					"median": 746.644,
					"mean": 731.091,
					"stddev": 49.424,
					"min": 633.623,
					"max": 770.449,
					"p25": 735.432,
					"p75": 754.782
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.281,
					"mean": 18.282,
					"stddev": 0.046,
					"min": 18.209,
					"max": 18.352,
					"p25": 18.275,
					"p75": 18.293
				},
				"ttft_ms": {
					"n": 6,
					"median": 2403.488,
					"mean": 2350.303,
					"stddev": 122.547,
					"min": 2183.516,
					"max": 2451.487,
					"p25": 2250.638,
					"p75": 2445.986
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 32635450512,
					"mean": 32633047517.333,
					"stddev": 4664242.117,
					"min": 32626939000,
					"max": 32636867728,
					"p25": 32629220496,
					"p75": 32636187774
				},
				"prefill_tps": {
					"n": 6,
					"median": 852.718,
					"mean": 873.857,
					"stddev": 46.834,
					"min": 835.821,
					"max": 938.446,
					"p25": 837.71,
					"p75": 911.228
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.476,
					"mean": 17.733,
					"stddev": 1.478,
					"min": 14.809,
					"max": 18.53,
					"p25": 17.809,
					"p75": 18.509
				},
				"ttft_ms": {
					"n": 6,
					"median": 2996.448,
					"mean": 3189.001,
					"stddev": 533.954,
					"min": 2756.41,
					"max": 4208.71,
					"p25": 2892.728,
					"p75": 3237.487
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 29683717672,
					"mean": 29683679450.667,
					"stddev": 152875.577,
					"min": 29683471912,
					"max": 29683865128,
					"p25": 29683566144,
					"p75": 29683770932
				},
				"prefill_tps": {
					"n": 6,
					"median": 683.706,
					"mean": 655.262,
					"stddev": 92.613,
					"min": 486.777,
					"max": 743.264,
					"p25": 633.957,
					"p75": 708.485
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.983,
					"mean": 18.976,
					"stddev": 0.04,
					"min": 18.902,
					"max": 19.015,
					"p25": 18.973,
					"p75": 18.999
				},
				"ttft_ms": {
					"n": 6,
					"median": 392.136,
					"mean": 401.907,
					"stddev": 24.312,
					"min": 391.414,
					"max": 451.531,
					"p25": 392.067,
					"p75": 392.17
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 29480465852,
					"mean": 29511931340,
					"stddev": 83543296.445,
					"min": 29469775304,
					"max": 29681587752,
					"p25": 29471356360,
					"p75": 29488346526
				},
				"prefill_tps": {
					"n": 6,
					"median": 655.745,
					"mean": 641.505,
					"stddev": 35.498,
					"min": 569.052,
					"max": 657.017,
					"p25": 655.652,
					"p75": 655.822
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"Its weights are 8-bit in another scheme: mlx-affine-8bit-g64, where most use Q8_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.352,
					"mean": 18.356,
					"stddev": 0.018,
					"min": 18.338,
					"max": 18.39,
					"p25": 18.348,
					"p75": 18.356
				},
				"ttft_ms": {
					"n": 6,
					"median": 428.314,
					"mean": 428.474,
					"stddev": 1.007,
					"min": 427.268,
					"max": 430.342,
					"p25": 428.232,
					"p75": 428.379
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 30039782520,
					"mean": 30039736146.667,
					"stddev": 793286.806,
					"min": 30038561912,
					"max": 30040544496,
					"p25": 30039299300,
					"p75": 30040401034
				},
				"prefill_tps": {
					"n": 6,
					"median": 599.125,
					"mean": 598.911,
					"stddev": 1.408,
					"min": 596.31,
					"max": 600.62,
					"p25": 599.062,
					"p75": 599.217
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-8-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 18.567,
					"mean": 18.574,
					"stddev": 0.029,
					"min": 18.538,
					"max": 18.625,
					"p25": 18.564,
					"p75": 18.578
				},
				"ttft_ms": {
					"n": 6,
					"median": 414.949,
					"mean": 412.666,
					"stddev": 6.109,
					"min": 400.753,
					"max": 417.49,
					"p25": 412.673,
					"p75": 415.693
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 29489313188,
					"mean": 29521622476,
					"stddev": 79209498.798,
					"min": 29489157528,
					"max": 29683308120,
					"p25": 29489276354,
					"p75": 29489362364
				},
				"prefill_tps": {
					"n": 6,
					"median": 618.13,
					"mean": 621.671,
					"stddev": 9.432,
					"min": 614.318,
					"max": 640.109,
					"p25": 617.008,
					"p75": 621.561
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 25.888,
					"mean": 25.873,
					"stddev": 0.046,
					"min": 25.781,
					"max": 25.905,
					"p25": 25.878,
					"p75": 25.895
				},
				"ttft_ms": {
					"n": 6,
					"median": 2822.549,
					"mean": 2896.69,
					"stddev": 221.453,
					"min": 2748.69,
					"max": 3328.011,
					"p25": 2757.867,
					"p75": 2897.162
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 20479786104,
					"mean": 20479898057.333,
					"stddev": 6011012.041,
					"min": 20474379384,
					"max": 20485536888,
					"p25": 20474428524,
					"p75": 20485389426
				},
				"prefill_tps": {
					"n": 6,
					"median": 727.048,
					"mean": 711.294,
					"stddev": 49.602,
					"min": 616.224,
					"max": 746.328,
					"p25": 708.045,
					"p75": 743.818
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 33.053,
					"mean": 33.058,
					"stddev": 0.016,
					"min": 33.048,
					"max": 33.091,
					"p25": 33.049,
					"p75": 33.056
				},
				"ttft_ms": {
					"n": 6,
					"median": 2144.985,
					"mean": 2165.367,
					"stddev": 103.355,
					"min": 2051.982,
					"max": 2288.444,
					"p25": 2087.232,
					"p75": 2256.616
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 19190099692,
					"mean": 19187830520,
					"stddev": 4072897.53,
					"min": 19182071568,
					"max": 19191246584,
					"p25": 19184893706,
					"p75": 19190255328
				},
				"prefill_tps": {
					"n": 6,
					"median": 955.991,
					"mean": 948.104,
					"stddev": 44.983,
					"min": 895.404,
					"max": 998.607,
					"p25": 908.214,
					"p75": 981.757
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 25.108,
					"mean": 25.089,
					"stddev": 0.118,
					"min": 24.94,
					"max": 25.265,
					"p25": 25.003,
					"p75": 25.136
				},
				"ttft_ms": {
					"n": 6,
					"median": 3091.639,
					"mean": 3086.938,
					"stddev": 202.241,
					"min": 2872.782,
					"max": 3345.053,
					"p25": 2908.404,
					"p75": 3227.8
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 20487044216,
					"mean": 20487066069.333,
					"stddev": 228379.593,
					"min": 20486847656,
					"max": 20487486656,
					"p25": 20486921318,
					"p75": 20487081044
				},
				"prefill_tps": {
					"n": 6,
					"median": 663.769,
					"mean": 666.032,
					"stddev": 43.486,
					"min": 612.434,
					"max": 713.14,
					"p25": 634.715,
					"p75": 704.481
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 25.973,
					"mean": 25.945,
					"stddev": 0.119,
					"min": 25.722,
					"max": 26.044,
					"p25": 25.932,
					"p75": 26.023
				},
				"ttft_ms": {
					"n": 6,
					"median": 394.421,
					"mean": 399.759,
					"stddev": 11.58,
					"min": 392.864,
					"max": 422.614,
					"p25": 393.446,
					"p75": 399.439
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 20291427316,
					"mean": 20320088409.333,
					"stddev": 81006238.238,
					"min": 20280507416,
					"max": 20485078136,
					"p25": 20283231190,
					"p75": 20291550172
				},
				"prefill_tps": {
					"n": 6,
					"median": 652.001,
					"mean": 643.671,
					"stddev": 17.841,
					"min": 608.604,
					"max": 654.405,
					"p25": 643.754,
					"p75": 653.506
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"Its weights are 4-bit in another scheme: mlx-affine-4bit-g64, where most use Q4_K_M."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 33.418,
					"mean": 33.424,
					"stddev": 0.037,
					"min": 33.387,
					"max": 33.486,
					"p25": 33.397,
					"p75": 33.437
				},
				"ttft_ms": {
					"n": 6,
					"median": 349.791,
					"mean": 349.749,
					"stddev": 0.414,
					"min": 349.147,
					"max": 350.226,
					"p25": 349.492,
					"p75": 350.06
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 16617033536,
					"mean": 16616806934.667,
					"stddev": 489374.926,
					"min": 16616124392,
					"max": 16617287416,
					"p25": 16616439574,
					"p75": 16617099240
				},
				"prefill_tps": {
					"n": 6,
					"median": 734.132,
					"mean": 734.211,
					"stddev": 0.882,
					"min": 733.188,
					"max": 735.493,
					"p25": 733.545,
					"p75": 734.76
				}
			}
		},
		{
			"experiment_id": "s2-qwen3.8-27b-4-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 25.114,
					"mean": 24.948,
					"stddev": 0.498,
					"min": 23.941,
					"max": 25.275,
					"p25": 25.087,
					"p75": 25.151
				},
				"ttft_ms": {
					"n": 6,
					"median": 420.68,
					"mean": 420.653,
					"stddev": 11.34,
					"min": 407.875,
					"max": 437.383,
					"p25": 411.436,
					"p75": 426.879
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 20292656116,
					"mean": 20325011797.333,
					"stddev": 79234901.613,
					"min": 20292582400,
					"max": 20486749280,
					"p25": 20292619228,
					"p75": 20292778984
				},
				"prefill_tps": {
					"n": 6,
					"median": 609.708,
					"mean": 610.092,
					"stddev": 16.398,
					"min": 586.321,
					"max": 628.844,
					"p25": 600.866,
					"p75": 623.466
				}
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-ollama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 44.588,
					"mean": 44.535,
					"stddev": 0.155,
					"min": 44.227,
					"max": 44.64,
					"p25": 44.548,
					"p75": 44.622
				},
				"ttft_ms": {
					"n": 6,
					"median": 2671.892,
					"mean": 2703.071,
					"stddev": 124.309,
					"min": 2571.585,
					"max": 2941.193,
					"p25": 2668.34,
					"p75": 2689.002
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 8714918824,
					"mean": 8714940657.333,
					"stddev": 160083.223,
					"min": 8714754960,
					"max": 8715213712,
					"p25": 8714853270,
					"p75": 8714984360
				},
				"prefill_tps": {
					"n": 6,
					"median": 767.817,
					"mean": 760.259,
					"stddev": 33.389,
					"min": 697.399,
					"max": 798.083,
					"p25": 762.904,
					"p75": 768.834
				}
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-mlx-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"Its weights are 2-bit in another scheme: mlx-affine-2bit-g128, where most use Q2_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 43.361,
					"mean": 42.906,
					"stddev": 1.242,
					"min": 40.407,
					"max": 43.681,
					"p25": 43.126,
					"p75": 43.522
				},
				"ttft_ms": {
					"n": 6,
					"median": 3291.336,
					"mean": 3274.571,
					"stddev": 185.652,
					"min": 2965.671,
					"max": 3481.506,
					"p25": 3209.272,
					"p75": 3399.577
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 15520192992,
					"mean": 15520152028,
					"stddev": 608698.217,
					"min": 15519406632,
					"max": 15521028576,
					"p25": 15519689130,
					"p75": 15520475598
				},
				"prefill_tps": {
					"n": 6,
					"median": 622.5,
					"mean": 627.421,
					"stddev": 36.869,
					"min": 588.476,
					"max": 690.886,
					"p25": 602.846,
					"p75": 638.526
				}
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-llama-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 42.72,
					"mean": 42.611,
					"stddev": 0.232,
					"min": 42.173,
					"max": 42.767,
					"p25": 42.574,
					"p75": 42.751
				},
				"ttft_ms": {
					"n": 6,
					"median": 2814.473,
					"mean": 2820.418,
					"stddev": 169.642,
					"min": 2574.599,
					"max": 3077.987,
					"p25": 2746.987,
					"p75": 2890.98
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 8716516216,
					"mean": 8716587245.333,
					"stddev": 228306.002,
					"min": 8716393360,
					"max": 8716999592,
					"p25": 8716430260,
					"p75": 8716651414
				},
				"prefill_tps": {
					"n": 6,
					"median": 728.09,
					"mean": 728.582,
					"stddev": 43.867,
					"min": 665.607,
					"max": 795.771,
					"p25": 708.676,
					"p75": 745.822
				}
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "unsupported",
				"detail": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
			},
			"approximate": [],
			"n_attempted": 0,
			"n_valid": 0,
			"metrics": {
				"decode_tps": null,
				"ttft_ms": null,
				"peak_memory_bytes": null,
				"prefill_tps": null
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-ollama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 45.125,
					"mean": 45.122,
					"stddev": 0.062,
					"min": 45.041,
					"max": 45.201,
					"p25": 45.076,
					"p75": 45.166
				},
				"ttft_ms": {
					"n": 6,
					"median": 375.03,
					"mean": 376.844,
					"stddev": 4.295,
					"min": 374.418,
					"max": 385.434,
					"p25": 374.517,
					"p75": 376.361
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 8521202468,
					"mean": 8553440746.667,
					"stddev": 79027611.086,
					"min": 8520997680,
					"max": 8714754984,
					"p25": 8521132854,
					"p75": 8521333540
				},
				"prefill_tps": {
					"n": 6,
					"median": 685.788,
					"mean": 682.587,
					"stddev": 7.646,
					"min": 667.318,
					"max": 686.975,
					"p25": 683.364,
					"p75": 686.768
				}
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-mlx-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"Its weights are 2-bit in another scheme: mlx-affine-2bit-g128, where most use Q2_0."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 45.288,
					"mean": 45.153,
					"stddev": 0.356,
					"min": 44.433,
					"max": 45.349,
					"p25": 45.235,
					"p75": 45.33
				},
				"ttft_ms": {
					"n": 6,
					"median": 421.433,
					"mean": 433.239,
					"stddev": 29.319,
					"min": 418.97,
					"max": 492.891,
					"p25": 419.638,
					"p75": 424.607
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 10358741568,
					"mean": 10357815840,
					"stddev": 2253412.688,
					"min": 10354891304,
					"max": 10359904784,
					"p25": 10355878350,
					"p75": 10359454296
				},
				"prefill_tps": {
					"n": 6,
					"median": 609.164,
					"mean": 594.56,
					"stddev": 36.369,
					"min": 520.647,
					"max": 612.692,
					"p25": 604.588,
					"p75": 611.762
				}
			}
		},
		{
			"experiment_id": "s2-bonsai2-27b-2-bit-llama-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 43.221,
					"mean": 42.896,
					"stddev": 1.067,
					"min": 40.75,
					"max": 43.535,
					"p25": 43.125,
					"p75": 43.474
				},
				"ttft_ms": {
					"n": 6,
					"median": 383.968,
					"mean": 388.14,
					"stddev": 9.383,
					"min": 382.385,
					"max": 406.681,
					"p25": 383.027,
					"p75": 387.833
				},
				"peak_memory_bytes": {
					"n": 6,
					"median": 8522226444,
					"mean": 8554956266.667,
					"stddev": 80211466.514,
					"min": 8522111792,
					"max": 8718687192,
					"p25": 8522197820,
					"p75": 8522279698
				},
				"prefill_tps": {
					"n": 6,
					"median": 668.089,
					"mean": 661.207,
					"stddev": 15.477,
					"min": 630.711,
					"max": 670.847,
					"p25": 661.462,
					"p75": 669.711
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-mlx-q4-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 2 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"The experiments here use different 4-bit schemes: Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 2,
			"metrics": {
				"decode_tps": {
					"n": 2,
					"median": 518.907,
					"mean": 518.907,
					"stddev": 1.003,
					"min": 518.198,
					"max": 519.617,
					"p25": 518.553,
					"p75": 519.262
				},
				"ttft_ms": {
					"n": 2,
					"median": 90.197,
					"mean": 90.197,
					"stddev": 0.906,
					"min": 89.557,
					"max": 90.838,
					"p25": 89.877,
					"p75": 90.518
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 2,
					"median": 22873.574,
					"mean": 22873.574,
					"stddev": 225.794,
					"min": 22713.913,
					"max": 23033.234,
					"p25": 22793.744,
					"p75": 22953.404
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-gguf-q4-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-2048-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "partial",
				"detail": "Partial: 3 of 6 planned trials were valid."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"The experiments here use different 4-bit schemes: Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 3,
			"metrics": {
				"decode_tps": {
					"n": 3,
					"median": 421.848,
					"mean": 424.952,
					"stddev": 9.879,
					"min": 416.998,
					"max": 436.011,
					"p25": 419.423,
					"p75": 428.93
				},
				"ttft_ms": {
					"n": 3,
					"median": 137.299,
					"mean": 137.617,
					"stddev": 0.579,
					"min": 137.267,
					"max": 138.285,
					"p25": 137.283,
					"p75": 137.792
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 3,
					"median": 15025.679,
					"mean": 15008.875,
					"stddev": 65.418,
					"min": 14936.694,
					"max": 15064.251,
					"p25": 14981.186,
					"p75": 15044.965
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-mlx-q4-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-256-128",
			"ranking_eligibility": {
				"eligible": false,
				"reason": "check-failed",
				"detail": "Its output check failed: 10/12; needs 11 and all mandatory checks."
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"The experiments here use different 4-bit schemes: Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 565.913,
					"mean": 561.869,
					"stddev": 9.612,
					"min": 543.567,
					"max": 569.004,
					"p25": 560.569,
					"p75": 567.503
				},
				"ttft_ms": {
					"n": 6,
					"median": 19.934,
					"mean": 19.946,
					"stddev": 0.64,
					"min": 19.339,
					"max": 21.121,
					"p25": 19.532,
					"p75": 19.947
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 13197.237,
					"mean": 13206.853,
					"stddev": 403.223,
					"min": 12486.587,
					"max": 13610.506,
					"p25": 13184.835,
					"p75": 13475.949
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-gguf-q4-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype.",
				"The experiments here use different 4-bit schemes: Q4_K_M, mlx-affine-4bit-g64."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 425.577,
					"mean": 428.031,
					"stddev": 8.987,
					"min": 416.413,
					"max": 440.931,
					"p25": 423.664,
					"p75": 433.892
				},
				"ttft_ms": {
					"n": 6,
					"median": 24.807,
					"mean": 25.668,
					"stddev": 2.171,
					"min": 23.926,
					"max": 29.666,
					"p25": 24.307,
					"p75": 26.197
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 10666.919,
					"mean": 10398.625,
					"stddev": 867.749,
					"min": 8874.099,
					"max": 11199.58,
					"p25": 10095.698,
					"p75": 10975.935
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-mlx-bf16-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 189.911,
					"mean": 189.761,
					"stddev": 2.053,
					"min": 187.052,
					"max": 192.756,
					"p25": 188.4,
					"p75": 190.759
				},
				"ttft_ms": {
					"n": 6,
					"median": 87.771,
					"mean": 88.103,
					"stddev": 0.675,
					"min": 87.602,
					"max": 89.273,
					"p25": 87.651,
					"p75": 88.384
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 23503.237,
					"mean": 23413.806,
					"stddev": 177.978,
					"min": 23105.736,
					"max": 23543.748,
					"p25": 23339.257,
					"p75": 23532.522
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-gguf-bf16-2048",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-2048-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 175.415,
					"mean": 171.32,
					"stddev": 10.555,
					"min": 156.665,
					"max": 182.025,
					"p25": 163.204,
					"p75": 178.306
				},
				"ttft_ms": {
					"n": 6,
					"median": 133.466,
					"mean": 133.368,
					"stddev": 0.75,
					"min": 132.361,
					"max": 134.193,
					"p25": 132.801,
					"p75": 133.971
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 15455.947,
					"mean": 15465.601,
					"stddev": 81.738,
					"min": 15369.26,
					"max": 15574.737,
					"p25": 15404.388,
					"p75": 15526.87
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-mlx-bf16-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 192.222,
					"mean": 192.387,
					"stddev": 1.485,
					"min": 190.689,
					"max": 194.652,
					"p25": 191.303,
					"p75": 193.209
				},
				"ttft_ms": {
					"n": 6,
					"median": 22.078,
					"mean": 22.063,
					"stddev": 0.327,
					"min": 21.523,
					"max": 22.422,
					"p25": 21.953,
					"p75": 22.295
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 11853.224,
					"mean": 11864.718,
					"stddev": 179.579,
					"min": 11660.214,
					"max": 12162.098,
					"p25": 11745.854,
					"p75": 11925.417
				}
			}
		},
		{
			"experiment_id": "liquid-mac-session1-gguf-bf16-256",
			"draft": false,
			"comparison_id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-256-128",
			"ranking_eligibility": {
				"eligible": true
			},
			"approximate": [
				"The experiments use different effective KV cache precision: f16, model-dtype."
			],
			"n_attempted": 6,
			"n_valid": 6,
			"metrics": {
				"decode_tps": {
					"n": 6,
					"median": 162.55,
					"mean": 160.444,
					"stddev": 13.293,
					"min": 140.641,
					"max": 173.593,
					"p25": 152.46,
					"p75": 171.312
				},
				"ttft_ms": {
					"n": 6,
					"median": 29.371,
					"mean": 29.068,
					"stddev": 2.212,
					"min": 26.434,
					"max": 31.923,
					"p25": 27.185,
					"p75": 30.482
				},
				"peak_memory_bytes": null,
				"prefill_tps": {
					"n": 6,
					"median": 8953.158,
					"mean": 9087.468,
					"stddev": 708.754,
					"min": 8215.925,
					"max": 9955.666,
					"p25": 8620.873,
					"p75": 9690.881
				}
			}
		}
	],
	"comparisons": [
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
				"s4-iphone12-lfm2.5-1.2b-4-bit-llama-2048",
				"s4-iphone12-lfm2.5-1.2b-4-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						},
						{
							"engine_id": "uzu",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-256",
				"s4-iphone12-lfm2.5-1.2b-4-bit-llama-256",
				"s4-iphone12-lfm2.5-1.2b-4-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": {
								"score": 94,
								"low": 89,
								"high": 94,
								"ratio": 0.93720232845299
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-256",
							"score": {
								"score": 83,
								"low": 83,
								"high": 83,
								"ratio": 0.8282986417357558
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-4-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-1.2b-8-bit-llama-2048",
				"s4-iphone12-lfm2.5-1.2b-8-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-1.2b-8-bit-llama-256",
				"s4-iphone12-lfm2.5-1.2b-8-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-256",
							"score": {
								"score": 94,
								"low": 92,
								"high": 94,
								"ratio": 0.9385831678155548
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-8-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-1.2b-bf16-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-1.2b-bf16-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "llama.cpp errored on the opposite check in qualification: The app wrote no result for run (exit 1); iOS ends an app that exceeds its memory limit or leaves the foreground."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-2.6b-4-bit-llama-2048",
				"s4-iphone12-lfm2.5-2.6b-4-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-2.6b-4-bit-llama-256",
				"s4-iphone12-lfm2.5-2.6b-4-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-4-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-2.6b-8-bit-llama-2048",
				"s4-iphone12-lfm2.5-2.6b-8-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-2.6b-8-bit-llama-256",
				"s4-iphone12-lfm2.5-2.6b-8-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "llama.cpp errored on the arithmetic check in qualification: The app failed: llama_decode(token 1) returned -3"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s4-iphone12-lfm2.5-2.6b-8-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-2048",
				"s4-iphone12-lfm2.5-350m-4-bit-llama-2048",
				"s4-iphone12-lfm2.5-350m-4-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "failed",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 98,
								"high": 112,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "uzu",
							"state": "failed",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "failed",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-256",
				"s4-iphone12-lfm2.5-350m-4-bit-llama-256",
				"s4-iphone12-lfm2.5-350m-4-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-350m--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-256",
							"score": {
								"score": 100,
								"low": 97,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-256",
							"score": {
								"score": 43,
								"low": 40,
								"high": 44,
								"ratio": 0.4278900905720946
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-4-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-350m-8-bit-llama-2048",
				"s4-iphone12-lfm2.5-350m-8-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-2048",
							"score": {
								"score": 100,
								"low": 98,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-2048",
							"score": {
								"score": 69,
								"low": 67,
								"high": 71,
								"ratio": 0.6885836812036075
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-350m-8-bit-llama-256",
				"s4-iphone12-lfm2.5-350m-8-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-350m--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-256",
							"score": {
								"score": 67,
								"low": 64,
								"high": 67,
								"ratio": 0.6661409324208726
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "uzu",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-8-bit-uzu-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-350m--bf16--archive-2048-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-350m-bf16-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-350m--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-iphone-12--ios-27--lfm2.5-350m--bf16--archive-256-128",
			"device": "glenn-iphone-12",
			"cohort": "ios-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "iOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-iphone12-lfm2.5-350m-bf16-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-iphone-12--ios-27--lfm2.5-350m--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-iphone12-lfm2.5-350m-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-1.2b-4-bit-et-mlx-2048",
				"s3-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
				"s3-lfm2.5-1.2b-4-bit-litert-cpu-2048",
				"s3-lfm2.5-1.2b-4-bit-litert-gpu-2048",
				"s3-lfm2.5-1.2b-4-bit-llama-2048",
				"s3-lfm2.5-1.2b-4-bit-magnitude-2048",
				"s3-lfm2.5-1.2b-4-bit-mlx-2048",
				"s3-lfm2.5-1.2b-4-bit-ollama-2048",
				"s3-lfm2.5-1.2b-4-bit-rapid-mlx-2048",
				"s3-lfm2.5-1.2b-4-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-2048",
							"score": {
								"score": 95,
								"low": 94,
								"high": 96,
								"ratio": 0.9525149417699219
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-2048",
							"score": {
								"score": 87,
								"low": 86,
								"high": 87,
								"ratio": 0.869478789884329
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-2048",
							"score": {
								"score": 85,
								"low": 83,
								"high": 86,
								"ratio": 0.852958519061651
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-2048",
							"score": {
								"score": 80,
								"low": 77,
								"high": 85,
								"ratio": 0.8024103252950882
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-2048",
							"score": {
								"score": 86,
								"low": 84,
								"high": 87,
								"ratio": 0.8641212854900032
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-2048",
							"score": {
								"score": 85,
								"low": 83,
								"high": 88,
								"ratio": 0.8514990447727316
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-2048",
							"score": {
								"score": 84,
								"low": 83,
								"high": 84,
								"ratio": 0.8362986362002762
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-2048",
							"score": {
								"score": 83,
								"low": 79,
								"high": 85,
								"ratio": 0.8260635178845026
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9987762009267462
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9978281639131286
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-2048",
							"score": {
								"score": 61,
								"low": 61,
								"high": 61,
								"ratio": 0.6067438183808185
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-2048",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-1.2b-4-bit-et-mlx-256",
				"s3-lfm2.5-1.2b-4-bit-et-xnnpack-256",
				"s3-lfm2.5-1.2b-4-bit-litert-cpu-256",
				"s3-lfm2.5-1.2b-4-bit-litert-gpu-256",
				"s3-lfm2.5-1.2b-4-bit-llama-256",
				"s3-lfm2.5-1.2b-4-bit-magnitude-256",
				"s3-lfm2.5-1.2b-4-bit-mlx-256",
				"s3-lfm2.5-1.2b-4-bit-ollama-256",
				"s3-lfm2.5-1.2b-4-bit-rapid-mlx-256",
				"s3-lfm2.5-1.2b-4-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-256",
							"score": {
								"score": 89,
								"low": 86,
								"high": 97,
								"ratio": 0.8899367786934386
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-256",
							"score": {
								"score": 85,
								"low": 83,
								"high": 86,
								"ratio": 0.8464896348752601
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-256",
							"score": {
								"score": 78,
								"low": 77,
								"high": 79,
								"ratio": 0.7796200007962897
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-256",
							"score": {
								"score": 71,
								"low": 69,
								"high": 71,
								"ratio": 0.7071141846945121
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-256",
							"score": {
								"score": 87,
								"low": 87,
								"high": 88,
								"ratio": 0.8731221970502566
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-256",
							"score": {
								"score": 84,
								"low": 84,
								"high": 85,
								"ratio": 0.8406676221739331
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-256",
							"score": {
								"score": 82,
								"low": 81,
								"high": 82,
								"ratio": 0.8221018077581025
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-256",
							"score": {
								"score": 81,
								"low": 80,
								"high": 82,
								"ratio": 0.8101456378011003
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-uzu-256",
							"score": {
								"score": 100,
								"low": 94,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-ollama-256",
							"score": {
								"score": 96,
								"low": 95,
								"high": 97,
								"ratio": 0.9645755936672373
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-llama-256",
							"score": {
								"score": 96,
								"low": 95,
								"high": 97,
								"ratio": 0.9636989848646231
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-magnitude-256",
							"score": {
								"score": 58,
								"low": 57,
								"high": 58,
								"ratio": 0.576399125778993
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-et-mlx-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-mlx-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-1.2b-4-bit-rapid-mlx-256",
							"score": null,
							"note": "Its output check failed: 10/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "liquid-2048-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.3",
			"experiment_ids": [
				"liquid-mac-session1-gguf-q4-2048",
				"liquid-mac-session1-mlx-q4-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-gguf-q4-2048",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-mlx-q4-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-gguf-q4-2048",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-mlx-q4-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-gguf-q4-2048",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-mlx-q4-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "liquid-256-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.3",
			"experiment_ids": [
				"liquid-mac-session1-gguf-q4-256",
				"liquid-mac-session1-mlx-q4-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--4-bit--liquid-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-gguf-q4-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-mlx-q4-256",
							"score": null,
							"note": "Its output check failed: 10/12; needs 11 and all mandatory checks."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "liquid-mac-session1-gguf-q4-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 104,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-mlx-q4-256",
							"score": null,
							"note": "Its output check failed: 10/12; needs 11 and all mandatory checks."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-gguf-q4-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "liquid-mac-session1-mlx-q4-256",
							"score": null,
							"note": "Its output check failed: 10/12; needs 11 and all mandatory checks."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-1.2b-8-bit-litert-2048",
				"s3-lfm2.5-1.2b-8-bit-llama-2048",
				"s3-lfm2.5-1.2b-8-bit-magnitude-2048",
				"s3-lfm2.5-1.2b-8-bit-mlx-2048",
				"s3-lfm2.5-1.2b-8-bit-ollama-2048",
				"s3-lfm2.5-1.2b-8-bit-rapid-mlx-2048",
				"s3-lfm2.5-1.2b-8-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-2048",
							"score": {
								"score": 89,
								"low": 89,
								"high": 90,
								"ratio": 0.8941043377164224
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-2048",
							"score": {
								"score": 88,
								"low": 87,
								"high": 88,
								"ratio": 0.8755453793053302
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-2048",
							"score": {
								"score": 86,
								"low": 85,
								"high": 86,
								"ratio": 0.8576963508980835
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-2048",
							"score": {
								"score": 84,
								"low": 84,
								"high": 85,
								"ratio": 0.8421972339101583
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-2048",
							"score": {
								"score": 81,
								"low": 80,
								"high": 81,
								"ratio": 0.8079237980138938
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-2048",
							"score": {
								"score": 78,
								"low": 78,
								"high": 80,
								"ratio": 0.780580946601121
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-2048",
							"score": {
								"score": 22,
								"low": 22,
								"high": 22,
								"ratio": 0.2224357294552374
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-2048",
							"score": {
								"score": 92,
								"low": 92,
								"high": 93,
								"ratio": 0.9235660844499042
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-2048",
							"score": {
								"score": 92,
								"low": 91,
								"high": 92,
								"ratio": 0.9177845933567088
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-2048",
							"score": {
								"score": 89,
								"low": 89,
								"high": 90,
								"ratio": 0.8938586571863231
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-2048",
							"score": {
								"score": 87,
								"low": 87,
								"high": 87,
								"ratio": 0.8714865645414536
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-2048",
							"score": {
								"score": 87,
								"low": 86,
								"high": 88,
								"ratio": 0.8665538570843605
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-2048",
							"score": {
								"score": 32,
								"low": 32,
								"high": 32,
								"ratio": 0.32212016932284426
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9982482255144389
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-2048",
							"score": {
								"score": 95,
								"low": 94,
								"high": 95,
								"ratio": 0.9455775758574938
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-2048",
							"score": {
								"score": 75,
								"low": 75,
								"high": 75,
								"ratio": 0.7508741103914061
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-2048",
							"score": {
								"score": 60,
								"low": 60,
								"high": 61,
								"ratio": 0.6006635640563548
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-2048",
							"score": {
								"score": 56,
								"low": 56,
								"high": 56,
								"ratio": 0.5616967373566564
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-2048",
							"score": {
								"score": 53,
								"low": 53,
								"high": 53,
								"ratio": 0.5296763853001988
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-1.2b-8-bit-litert-256",
				"s3-lfm2.5-1.2b-8-bit-llama-256",
				"s3-lfm2.5-1.2b-8-bit-magnitude-256",
				"s3-lfm2.5-1.2b-8-bit-mlx-256",
				"s3-lfm2.5-1.2b-8-bit-ollama-256",
				"s3-lfm2.5-1.2b-8-bit-rapid-mlx-256",
				"s3-lfm2.5-1.2b-8-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-256",
							"score": {
								"score": 92,
								"low": 92,
								"high": 93,
								"ratio": 0.9216547657684537
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-256",
							"score": {
								"score": 90,
								"low": 85,
								"high": 98,
								"ratio": 0.8999558416236253
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-256",
							"score": {
								"score": 87,
								"low": 86,
								"high": 87,
								"ratio": 0.8688571265476782
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-256",
							"score": {
								"score": 86,
								"low": 81,
								"high": 87,
								"ratio": 0.8616448333386717
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-256",
							"score": {
								"score": 85,
								"low": 82,
								"high": 87,
								"ratio": 0.8499642384541263
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-256",
							"score": {
								"score": 69,
								"low": 68,
								"high": 69,
								"ratio": 0.6897888680616915
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-256",
							"score": {
								"score": 29,
								"low": 29,
								"high": 29,
								"ratio": 0.2879809875291181
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-256",
							"score": {
								"score": 97,
								"low": 96,
								"high": 97,
								"ratio": 0.9670532849192163
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-256",
							"score": {
								"score": 96,
								"low": 96,
								"high": 97,
								"ratio": 0.9647020474469035
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-256",
							"score": {
								"score": 88,
								"low": 87,
								"high": 88,
								"ratio": 0.8818957386492674
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-256",
							"score": {
								"score": 87,
								"low": 79,
								"high": 89,
								"ratio": 0.8714380756190033
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-256",
							"score": {
								"score": 87,
								"low": 77,
								"high": 88,
								"ratio": 0.8684242166772205
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-256",
							"score": {
								"score": 32,
								"low": 31,
								"high": 32,
								"ratio": 0.31810639883934366
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9989622717459912
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-mlx-256",
							"score": {
								"score": 98,
								"low": 98,
								"high": 99,
								"ratio": 0.9774515654042127
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-uzu-256",
							"score": {
								"score": 97,
								"low": 93,
								"high": 97,
								"ratio": 0.9672169694277089
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-rapid-mlx-256",
							"score": {
								"score": 84,
								"low": 83,
								"high": 84,
								"ratio": 0.8352160489893202
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-litert-256",
							"score": {
								"score": 81,
								"low": 81,
								"high": 82,
								"ratio": 0.8148977693997924
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-8-bit-magnitude-256",
							"score": {
								"score": 74,
								"low": 74,
								"high": 75,
								"ratio": 0.7437515072252614
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-1.2b-bf16-llama-2048",
				"s3-lfm2.5-1.2b-bf16-magnitude-2048",
				"s3-lfm2.5-1.2b-bf16-mlx-2048",
				"s3-lfm2.5-1.2b-bf16-ollama-2048",
				"s3-lfm2.5-1.2b-bf16-rapid-mlx-2048",
				"s3-lfm2.5-1.2b-bf16-zml-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-2048",
							"score": {
								"score": 91,
								"low": 90,
								"high": 91,
								"ratio": 0.907411264929505
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-2048",
							"score": {
								"score": 88,
								"low": 87,
								"high": 89,
								"ratio": 0.8825038039022804
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-2048",
							"score": {
								"score": 88,
								"low": 87,
								"high": 89,
								"ratio": 0.8819065307032509
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-2048",
							"score": {
								"score": 87,
								"low": 86,
								"high": 88,
								"ratio": 0.8665234807204538
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-2048",
							"score": {
								"score": 85,
								"low": 83,
								"high": 86,
								"ratio": 0.8491445877916617
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-2048",
							"score": {
								"score": 66,
								"low": 64,
								"high": 66,
								"ratio": 0.6562844512079593
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-2048",
							"score": {
								"score": 99,
								"low": 99,
								"high": 100,
								"ratio": 0.9936676519212464
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-2048",
							"score": {
								"score": 99,
								"low": 96,
								"high": 99,
								"ratio": 0.9898001102149191
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-2048",
							"score": {
								"score": 98,
								"low": 96,
								"high": 99,
								"ratio": 0.9772806973598516
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-2048",
							"score": {
								"score": 92,
								"low": 87,
								"high": 95,
								"ratio": 0.9228746054806873
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-2048",
							"score": {
								"score": 76,
								"low": 73,
								"high": 76,
								"ratio": 0.7557286709082711
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9990969932679473
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-2048",
							"score": {
								"score": 83,
								"low": 83,
								"high": 83,
								"ratio": 0.827199681472682
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-2048",
							"score": {
								"score": 75,
								"low": 75,
								"high": 75,
								"ratio": 0.7519194993423656
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-2048",
							"score": {
								"score": 71,
								"low": 71,
								"high": 72,
								"ratio": 0.7089131712331338
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-2048",
							"score": {
								"score": 52,
								"low": 50,
								"high": 52,
								"ratio": 0.5153915194297435
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-1.2b-bf16-llama-256",
				"s3-lfm2.5-1.2b-bf16-magnitude-256",
				"s3-lfm2.5-1.2b-bf16-mlx-256",
				"s3-lfm2.5-1.2b-bf16-ollama-256",
				"s3-lfm2.5-1.2b-bf16-rapid-mlx-256",
				"s3-lfm2.5-1.2b-bf16-zml-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-256",
							"score": {
								"score": 93,
								"low": 93,
								"high": 94,
								"ratio": 0.9311509977653083
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-256",
							"score": {
								"score": 90,
								"low": 89,
								"high": 91,
								"ratio": 0.9034789985982449
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-256",
							"score": {
								"score": 89,
								"low": 86,
								"high": 102,
								"ratio": 0.89449736981738
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-256",
							"score": {
								"score": 88,
								"low": 86,
								"high": 90,
								"ratio": 0.8795964500314405
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-256",
							"score": {
								"score": 87,
								"low": 83,
								"high": 88,
								"ratio": 0.8654709130099854
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-256",
							"score": {
								"score": 78,
								"low": 70,
								"high": 79,
								"ratio": 0.7808968897173827
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 0.9996193052320486
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 0.9966589024970571
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-256",
							"score": {
								"score": 99,
								"low": 97,
								"high": 100,
								"ratio": 0.9919453001728153
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-256",
							"score": {
								"score": 96,
								"low": 74,
								"high": 97,
								"ratio": 0.9647857339644852
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-256",
							"score": {
								"score": 95,
								"low": 89,
								"high": 97,
								"ratio": 0.9544468655295916
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-ollama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9989845548226013
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-mlx-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9961093059321063
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-rapid-mlx-256",
							"score": {
								"score": 90,
								"low": 90,
								"high": 91,
								"ratio": 0.9003827153315516
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-magnitude-256",
							"score": {
								"score": 82,
								"low": 82,
								"high": 83,
								"ratio": 0.8237391775545374
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-1.2b-bf16-zml-256",
							"score": {
								"score": 49,
								"low": 49,
								"high": 51,
								"ratio": 0.49357164952989496
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "liquid-2048-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.3",
			"experiment_ids": [
				"liquid-mac-session1-gguf-bf16-2048",
				"liquid-mac-session1-mlx-bf16-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-gguf-bf16-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "mlx-lm",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-mlx-bf16-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "liquid-mac-session1-mlx-bf16-2048",
							"score": {
								"score": 100,
								"low": 98,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "liquid-mac-session1-gguf-bf16-2048",
							"score": {
								"score": 92,
								"low": 82,
								"high": 96,
								"ratio": 0.9236695083486475
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-gguf-bf16-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "mlx-lm",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-mlx-bf16-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "liquid-256-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.3",
			"experiment_ids": [
				"liquid-mac-session1-gguf-bf16-256",
				"liquid-mac-session1-mlx-bf16-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-1.2b-instruct--bf16--liquid-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-gguf-bf16-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "mlx-lm",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-mlx-bf16-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "liquid-mac-session1-mlx-bf16-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "liquid-mac-session1-gguf-bf16-256",
							"score": {
								"score": 85,
								"low": 73,
								"high": 90,
								"ratio": 0.845636815765105
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-gguf-bf16-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "mlx-lm",
							"state": "missing",
							"experiment_id": "liquid-mac-session1-mlx-bf16-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-2.6b-4-bit-litert-2048",
				"s3-lfm2.5-2.6b-4-bit-llama-2048",
				"s3-lfm2.5-2.6b-4-bit-magnitude-2048",
				"s3-lfm2.5-2.6b-4-bit-mlx-2048",
				"s3-lfm2.5-2.6b-4-bit-ollama-2048",
				"s3-lfm2.5-2.6b-4-bit-rapid-mlx-2048",
				"s3-lfm2.5-2.6b-4-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-2048",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "ollama",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-2048",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-2048",
							"score": {
								"score": 100,
								"low": 97,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "ollama",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-2048",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "ollama",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-2048",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-2.6b-4-bit-litert-256",
				"s3-lfm2.5-2.6b-4-bit-llama-256",
				"s3-lfm2.5-2.6b-4-bit-magnitude-256",
				"s3-lfm2.5-2.6b-4-bit-mlx-256",
				"s3-lfm2.5-2.6b-4-bit-ollama-256",
				"s3-lfm2.5-2.6b-4-bit-rapid-mlx-256",
				"s3-lfm2.5-2.6b-4-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "ollama",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "ollama",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-litert-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "mlx-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-mlx-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "ollama",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-ollama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "rapid-mlx",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-rapid-mlx-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-4-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-2.6b-8-bit-litert-2048",
				"s3-lfm2.5-2.6b-8-bit-llama-2048",
				"s3-lfm2.5-2.6b-8-bit-magnitude-2048",
				"s3-lfm2.5-2.6b-8-bit-mlx-2048",
				"s3-lfm2.5-2.6b-8-bit-ollama-2048",
				"s3-lfm2.5-2.6b-8-bit-rapid-mlx-2048",
				"s3-lfm2.5-2.6b-8-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-2048",
							"score": {
								"score": 91,
								"low": 91,
								"high": 91,
								"ratio": 0.9079388933913903
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-2048",
							"score": {
								"score": 91,
								"low": 90,
								"high": 91,
								"ratio": 0.9053862953324082
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-2048",
							"score": {
								"score": 87,
								"low": 87,
								"high": 88,
								"ratio": 0.8697095489285173
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-2048",
							"score": {
								"score": 87,
								"low": 86,
								"high": 88,
								"ratio": 0.8689164706837197
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.998035694241778
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-2048",
							"score": {
								"score": 96,
								"low": 96,
								"high": 96,
								"ratio": 0.9576208827527719
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-2048",
							"score": {
								"score": 89,
								"low": 88,
								"high": 92,
								"ratio": 0.8879535051944976
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9994333686249705
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-2048",
							"score": {
								"score": 74,
								"low": 74,
								"high": 76,
								"ratio": 0.742167186804909
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-2048",
							"score": {
								"score": 72,
								"low": 71,
								"high": 73,
								"ratio": 0.7191288670824043
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-2.6b-8-bit-litert-256",
				"s3-lfm2.5-2.6b-8-bit-llama-256",
				"s3-lfm2.5-2.6b-8-bit-magnitude-256",
				"s3-lfm2.5-2.6b-8-bit-mlx-256",
				"s3-lfm2.5-2.6b-8-bit-ollama-256",
				"s3-lfm2.5-2.6b-8-bit-rapid-mlx-256",
				"s3-lfm2.5-2.6b-8-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-256",
							"score": {
								"score": 99,
								"low": 96,
								"high": 100,
								"ratio": 0.9895015778964015
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-256",
							"score": {
								"score": 98,
								"low": 98,
								"high": 99,
								"ratio": 0.983373563354343
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-256",
							"score": {
								"score": 92,
								"low": 90,
								"high": 94,
								"ratio": 0.9178545232862322
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-256",
							"score": {
								"score": 91,
								"low": 84,
								"high": 93,
								"ratio": 0.9148963427162005
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-256",
							"score": {
								"score": 100,
								"low": 97,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9971061408671767
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-256",
							"score": {
								"score": 95,
								"low": 95,
								"high": 95,
								"ratio": 0.9509454090641554
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-256",
							"score": {
								"score": 91,
								"low": 71,
								"high": 92,
								"ratio": 0.912497547577006
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9996544398458721
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-mlx-256",
							"score": {
								"score": 98,
								"low": 96,
								"high": 99,
								"ratio": 0.978946935470994
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-rapid-mlx-256",
							"score": {
								"score": 90,
								"low": 87,
								"high": 92,
								"ratio": 0.901346838906969
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s3-lfm2.5-2.6b-8-bit-uzu-256",
							"score": null,
							"note": "uzu can't run the frozen prompts: Error: uzu counted 23 input tokens, not the frozen prompt's 26 (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-lfm2.5-2.6b-bf16-llama-2048",
				"s2-lfm2.5-2.6b-bf16-magnitude-2048",
				"s2-lfm2.5-2.6b-bf16-ollama-2048",
				"s2-lfm2.5-2.6b-bf16-zml-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-2048",
							"score": {
								"score": 97,
								"low": 96,
								"high": 98,
								"ratio": 0.9733845858778882
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "zml",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-2048",
							"score": null,
							"note": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-2048",
							"score": {
								"score": 97,
								"low": 94,
								"high": 98,
								"ratio": 0.9740425158116655
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "zml",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-2048",
							"score": null,
							"note": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9996923400986748
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-2048",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "zml",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-2048",
							"score": null,
							"note": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-lfm2.5-2.6b-bf16-llama-256",
				"s2-lfm2.5-2.6b-bf16-magnitude-256",
				"s2-lfm2.5-2.6b-bf16-ollama-256",
				"s2-lfm2.5-2.6b-bf16-zml-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-2.6b--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-256",
							"score": {
								"score": 93,
								"low": 92,
								"high": 93,
								"ratio": 0.9266301068285773
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "zml",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-256",
							"score": null,
							"note": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-256",
							"score": {
								"score": 96,
								"low": 95,
								"high": 97,
								"ratio": 0.9607488054458578
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "zml",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-256",
							"score": null,
							"note": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-ollama-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-llama-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9998679249398541
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-magnitude-256",
							"score": null,
							"note": "Magnitude's chat API renders LFM2.5 2.6B's own template, which always opens a thinking block, so it streams reasoning and can't run the closed-think prompts (measured in qualification)."
						},
						{
							"engine_id": "zml",
							"state": "unsupported",
							"experiment_id": "s2-lfm2.5-2.6b-bf16-zml-256",
							"score": null,
							"note": "zml can't run the frozen prompts: Error: LLMD streamed reasoning before its answer; the harness can't time it apart from the answer (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-350m-4-bit-et-mlx-2048",
				"s3-lfm2.5-350m-4-bit-et-xnnpack-2048",
				"s3-lfm2.5-350m-4-bit-litert-2048",
				"s3-lfm2.5-350m-4-bit-llama-2048",
				"s3-lfm2.5-350m-4-bit-magnitude-2048",
				"s3-lfm2.5-350m-4-bit-mlx-2048",
				"s3-lfm2.5-350m-4-bit-ollama-2048",
				"s3-lfm2.5-350m-4-bit-rapid-mlx-2048",
				"s3-lfm2.5-350m-4-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-2048",
							"score": {
								"score": 81,
								"low": 81,
								"high": 87,
								"ratio": 0.812777436169923
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-llama-2048",
							"score": {
								"score": 79,
								"low": 76,
								"high": 85,
								"ratio": 0.7868979146862274
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-2048",
							"score": {
								"score": 75,
								"low": 73,
								"high": 75,
								"ratio": 0.7454309221658393
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-2048",
							"score": {
								"score": 59,
								"low": 55,
								"high": 82,
								"ratio": 0.5907099782254793
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-2048",
							"score": {
								"score": 59,
								"low": 56,
								"high": 63,
								"ratio": 0.590495180160509
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "uzu",
							"state": "failed",
							"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-2048",
							"score": {
								"score": 95,
								"low": 95,
								"high": 95,
								"ratio": 0.9482727481086151
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-2048",
							"score": {
								"score": 93,
								"low": 93,
								"high": 96,
								"ratio": 0.9346488620004584
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-2048",
							"score": {
								"score": 74,
								"low": 73,
								"high": 77,
								"ratio": 0.7399431554732661
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-llama-2048",
							"score": {
								"score": 69,
								"low": 68,
								"high": 73,
								"ratio": 0.6864816279799281
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "uzu",
							"state": "failed",
							"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9959637184125321
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-2048",
							"score": {
								"score": 44,
								"low": 44,
								"high": 44,
								"ratio": 0.4431734469185035
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-2048",
							"score": {
								"score": 26,
								"low": 22,
								"high": 59,
								"ratio": 0.2628450144877921
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-2048",
							"score": {
								"score": 25,
								"low": 22,
								"high": 30,
								"ratio": 0.25491679703853676
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "uzu",
							"state": "failed",
							"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-350m-4-bit-et-mlx-256",
				"s3-lfm2.5-350m-4-bit-et-xnnpack-256",
				"s3-lfm2.5-350m-4-bit-litert-256",
				"s3-lfm2.5-350m-4-bit-llama-256",
				"s3-lfm2.5-350m-4-bit-magnitude-256",
				"s3-lfm2.5-350m-4-bit-mlx-256",
				"s3-lfm2.5-350m-4-bit-ollama-256",
				"s3-lfm2.5-350m-4-bit-rapid-mlx-256",
				"s3-lfm2.5-350m-4-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-350m--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-256",
							"score": {
								"score": 90,
								"low": 90,
								"high": 91,
								"ratio": 0.9016658699868727
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-256",
							"score": {
								"score": 84,
								"low": 83,
								"high": 84,
								"ratio": 0.8365048078554397
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-llama-256",
							"score": {
								"score": 79,
								"low": 77,
								"high": 98,
								"ratio": 0.7934803675520627
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-256",
							"score": {
								"score": 79,
								"low": 76,
								"high": 81,
								"ratio": 0.7858389327557322
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-256",
							"score": {
								"score": 71,
								"low": 64,
								"high": 72,
								"ratio": 0.7108030681816324
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-256",
							"score": {
								"score": 62,
								"low": 60,
								"high": 62,
								"ratio": 0.6179912317829305
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.995359932534376
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-256",
							"score": {
								"score": 90,
								"low": 90,
								"high": 91,
								"ratio": 0.903706185041087
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-256",
							"score": {
								"score": 82,
								"low": 82,
								"high": 83,
								"ratio": 0.823361416922567
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-256",
							"score": {
								"score": 67,
								"low": 66,
								"high": 67,
								"ratio": 0.6707204429327682
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-llama-256",
							"score": {
								"score": 62,
								"low": 60,
								"high": 65,
								"ratio": 0.6184336815265
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-llama-256",
							"score": {
								"score": 100,
								"low": 97,
								"high": 100,
								"ratio": 0.9998167283065394
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-uzu-256",
							"score": {
								"score": 95,
								"low": 87,
								"high": 95,
								"ratio": 0.9476980330219054
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-mlx-256",
							"score": {
								"score": 91,
								"low": 91,
								"high": 92,
								"ratio": 0.9137929969888453
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-rapid-mlx-256",
							"score": {
								"score": 64,
								"low": 62,
								"high": 67,
								"ratio": 0.6352711749420085
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-4-bit-magnitude-256",
							"score": {
								"score": 44,
								"low": 43,
								"high": 44,
								"ratio": 0.4361715849495344
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-et-mlx-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s3-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-350m-8-bit-litert-2048",
				"s3-lfm2.5-350m-8-bit-llama-2048",
				"s3-lfm2.5-350m-8-bit-magnitude-2048",
				"s3-lfm2.5-350m-8-bit-mlx-2048",
				"s3-lfm2.5-350m-8-bit-ollama-2048",
				"s3-lfm2.5-350m-8-bit-rapid-mlx-2048",
				"s3-lfm2.5-350m-8-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-2048",
							"score": {
								"score": 82,
								"low": 82,
								"high": 83,
								"ratio": 0.8248583749261186
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-2048",
							"score": {
								"score": 82,
								"low": 82,
								"high": 83,
								"ratio": 0.823087638320751
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-llama-2048",
							"score": {
								"score": 81,
								"low": 77,
								"high": 83,
								"ratio": 0.8106100815861402
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-2048",
							"score": {
								"score": 76,
								"low": 75,
								"high": 76,
								"ratio": 0.7558996270162999
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-2048",
							"score": {
								"score": 59,
								"low": 56,
								"high": 60,
								"ratio": 0.5872546106905645
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-2048",
							"score": {
								"score": 58,
								"low": 54,
								"high": 60,
								"ratio": 0.579740154331381
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-litert-2048",
							"score": {
								"score": 21,
								"low": 20,
								"high": 21,
								"ratio": 0.20564332962235546
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-2048",
							"score": {
								"score": 96,
								"low": 94,
								"high": 98,
								"ratio": 0.963738908804148
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-2048",
							"score": {
								"score": 92,
								"low": 92,
								"high": 92,
								"ratio": 0.9177634376457672
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-2048",
							"score": {
								"score": 90,
								"low": 90,
								"high": 91,
								"ratio": 0.9047581004708363
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-2048",
							"score": {
								"score": 77,
								"low": 76,
								"high": 78,
								"ratio": 0.7728724103362342
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-llama-2048",
							"score": {
								"score": 76,
								"low": 73,
								"high": 79,
								"ratio": 0.7630098471006331
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-litert-2048",
							"score": {
								"score": 26,
								"low": 26,
								"high": 26,
								"ratio": 0.2562501127481425
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9969667231366682
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-2048",
							"score": {
								"score": 86,
								"low": 86,
								"high": 86,
								"ratio": 0.8632249819115192
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-2048",
							"score": {
								"score": 58,
								"low": 58,
								"high": 58,
								"ratio": 0.582342882304644
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-litert-2048",
							"score": {
								"score": 49,
								"low": 48,
								"high": 49,
								"ratio": 0.48551554000893765
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-2048",
							"score": {
								"score": 30,
								"low": 26,
								"high": 32,
								"ratio": 0.2957951543885322
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-2048",
							"score": {
								"score": 29,
								"low": 25,
								"high": 31,
								"ratio": 0.2909522455542233
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-350m-8-bit-litert-256",
				"s3-lfm2.5-350m-8-bit-llama-256",
				"s3-lfm2.5-350m-8-bit-magnitude-256",
				"s3-lfm2.5-350m-8-bit-mlx-256",
				"s3-lfm2.5-350m-8-bit-ollama-256",
				"s3-lfm2.5-350m-8-bit-rapid-mlx-256",
				"s3-lfm2.5-350m-8-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-350m--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-256",
							"score": {
								"score": 88,
								"low": 87,
								"high": 89,
								"ratio": 0.8781859930184698
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-256",
							"score": {
								"score": 87,
								"low": 86,
								"high": 88,
								"ratio": 0.8663888705430316
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-llama-256",
							"score": {
								"score": 84,
								"low": 82,
								"high": 87,
								"ratio": 0.8443480064791069
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-256",
							"score": {
								"score": 81,
								"low": 80,
								"high": 82,
								"ratio": 0.8129192260207052
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-256",
							"score": {
								"score": 74,
								"low": 71,
								"high": 79,
								"ratio": 0.7369630606032002
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-256",
							"score": {
								"score": 61,
								"low": 59,
								"high": 62,
								"ratio": 0.6091477073097227
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-litert-256",
							"score": {
								"score": 29,
								"low": 29,
								"high": 29,
								"ratio": 0.2916687082594164
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-256",
							"score": {
								"score": 98,
								"low": 97,
								"high": 100,
								"ratio": 0.9767117965709093
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-256",
							"score": {
								"score": 97,
								"low": 97,
								"high": 97,
								"ratio": 0.970495744790025
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-256",
							"score": {
								"score": 93,
								"low": 93,
								"high": 94,
								"ratio": 0.9342342210265772
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-256",
							"score": {
								"score": 77,
								"low": 76,
								"high": 79,
								"ratio": 0.7700744997715406
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-llama-256",
							"score": {
								"score": 75,
								"low": 72,
								"high": 80,
								"ratio": 0.7540285830871591
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-litert-256",
							"score": {
								"score": 24,
								"low": 24,
								"high": 25,
								"ratio": 0.24432333242926924
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 0.9989278510237696
							},
							"note": null
						},
						{
							"engine_id": "uzu",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-uzu-256",
							"score": {
								"score": 89,
								"low": 84,
								"high": 89,
								"ratio": 0.8929629897496288
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-mlx-256",
							"score": {
								"score": 89,
								"low": 88,
								"high": 92,
								"ratio": 0.8855174403655531
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-litert-256",
							"score": {
								"score": 77,
								"low": 77,
								"high": 77,
								"ratio": 0.7668267767461594
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-rapid-mlx-256",
							"score": {
								"score": 66,
								"low": 65,
								"high": 68,
								"ratio": 0.6642452434409518
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-8-bit-magnitude-256",
							"score": {
								"score": 58,
								"low": 57,
								"high": 58,
								"ratio": 0.575024585412107
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-350m-bf16-llama-2048",
				"s3-lfm2.5-350m-bf16-magnitude-2048",
				"s3-lfm2.5-350m-bf16-mlx-2048",
				"s3-lfm2.5-350m-bf16-ollama-2048",
				"s3-lfm2.5-350m-bf16-rapid-mlx-2048",
				"s3-lfm2.5-350m-bf16-zml-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-mlx-2048",
							"score": {
								"score": 96,
								"low": 96,
								"high": 106,
								"ratio": 0.9638837308375374
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-2048",
							"score": {
								"score": 84,
								"low": 81,
								"high": 85,
								"ratio": 0.8382136467079763
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-ollama-2048",
							"score": {
								"score": 81,
								"low": 79,
								"high": 82,
								"ratio": 0.8133827576807603
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-2048",
							"score": {
								"score": 80,
								"low": 79,
								"high": 81,
								"ratio": 0.8022476675296162
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-llama-2048",
							"score": {
								"score": 79,
								"low": 76,
								"high": 80,
								"ratio": 0.7858235500970676
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-zml-2048",
							"score": {
								"score": 39,
								"low": 39,
								"high": 40,
								"ratio": 0.39114842078858836
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-mlx-2048",
							"score": {
								"score": 99,
								"low": 99,
								"high": 99,
								"ratio": 0.9899873550195688
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-2048",
							"score": {
								"score": 96,
								"low": 92,
								"high": 98,
								"ratio": 0.9583861773265919
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-ollama-2048",
							"score": {
								"score": 88,
								"low": 81,
								"high": 89,
								"ratio": 0.8811503362287144
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-llama-2048",
							"score": {
								"score": 83,
								"low": 80,
								"high": 86,
								"ratio": 0.8317447254648863
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-zml-2048",
							"score": {
								"score": 49,
								"low": 49,
								"high": 49,
								"ratio": 0.49040916751686786
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9974987522245302
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-mlx-2048",
							"score": {
								"score": 90,
								"low": 90,
								"high": 93,
								"ratio": 0.9045744202192177
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-2048",
							"score": {
								"score": 71,
								"low": 69,
								"high": 72,
								"ratio": 0.713686215619057
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-2048",
							"score": {
								"score": 68,
								"low": 68,
								"high": 68,
								"ratio": 0.6803665135068053
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-zml-2048",
							"score": {
								"score": 19,
								"low": 19,
								"high": 19,
								"ratio": 0.1879535518833484
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s3-lfm2.5-350m-bf16-llama-256",
				"s3-lfm2.5-350m-bf16-magnitude-256",
				"s3-lfm2.5-350m-bf16-mlx-256",
				"s3-lfm2.5-350m-bf16-ollama-256",
				"s3-lfm2.5-350m-bf16-rapid-mlx-256",
				"s3-lfm2.5-350m-bf16-zml-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--lfm2.5-350m--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-mlx-256",
							"score": {
								"score": 95,
								"low": 94,
								"high": 119,
								"ratio": 0.9453893286978559
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-ollama-256",
							"score": {
								"score": 84,
								"low": 84,
								"high": 85,
								"ratio": 0.843295024592952
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-llama-256",
							"score": {
								"score": 82,
								"low": 77,
								"high": 98,
								"ratio": 0.8204081646066105
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-256",
							"score": {
								"score": 80,
								"low": 79,
								"high": 81,
								"ratio": 0.8021472551638917
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-256",
							"score": {
								"score": 77,
								"low": 76,
								"high": 85,
								"ratio": 0.7729278800714863
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-zml-256",
							"score": {
								"score": 51,
								"low": 51,
								"high": 53,
								"ratio": 0.5148039394021067
							},
							"note": null
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-mlx-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-256",
							"score": {
								"score": 98,
								"low": 97,
								"high": 98,
								"ratio": 0.977857345260293
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-256",
							"score": {
								"score": 92,
								"low": 90,
								"high": 92,
								"ratio": 0.9166751209546233
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-ollama-256",
							"score": {
								"score": 84,
								"low": 83,
								"high": 85,
								"ratio": 0.8386694466697685
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-llama-256",
							"score": {
								"score": 81,
								"low": 77,
								"high": 84,
								"ratio": 0.811980717226807
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-zml-256",
							"score": {
								"score": 73,
								"low": 73,
								"high": 74,
								"ratio": 0.734947577584126
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-ollama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9989326448019353
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-mlx-256",
							"score": {
								"score": 96,
								"low": 96,
								"high": 99,
								"ratio": 0.9626486773560836
							},
							"note": null
						},
						{
							"engine_id": "rapid-mlx",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-rapid-mlx-256",
							"score": {
								"score": 78,
								"low": 77,
								"high": 80,
								"ratio": 0.7776920245335075
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-magnitude-256",
							"score": {
								"score": 67,
								"low": 67,
								"high": 68,
								"ratio": 0.674096206807495
							},
							"note": null
						},
						{
							"engine_id": "zml",
							"state": "scored",
							"experiment_id": "s3-lfm2.5-350m-bf16-zml-256",
							"score": {
								"score": 19,
								"low": 18,
								"high": 19,
								"ratio": 0.1856390006946364
							},
							"note": null
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "qwen3.8-27b",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-qwen3.8-27b-4-bit-llama-2048",
				"s2-qwen3.8-27b-4-bit-magnitude-2048",
				"s2-qwen3.8-27b-4-bit-mlx-2048",
				"s2-qwen3.8-27b-4-bit-ollama-2048",
				"s2-qwen3.8-27b-4-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-2048",
							"score": {
								"score": 100,
								"low": 98,
								"high": 102,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-2048",
							"score": {
								"score": 82,
								"low": 78,
								"high": 83,
								"ratio": 0.8231406827610427
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-llama-2048",
							"score": {
								"score": 79,
								"low": 77,
								"high": 81,
								"ratio": 0.7903356063183264
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-2048",
							"score": {
								"score": 78,
								"low": 78,
								"high": 78,
								"ratio": 0.7832269385532328
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-llama-2048",
							"score": {
								"score": 76,
								"low": 75,
								"high": 76,
								"ratio": 0.7596284754787765
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-2048",
							"score": {
								"score": 94,
								"low": 94,
								"high": 94,
								"ratio": 0.937026372958646
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-llama-2048",
							"score": {
								"score": 94,
								"low": 94,
								"high": 94,
								"ratio": 0.9366944049943959
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-2048",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "qwen3.8-27b",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-qwen3.8-27b-4-bit-llama-256",
				"s2-qwen3.8-27b-4-bit-magnitude-256",
				"s2-qwen3.8-27b-4-bit-mlx-256",
				"s2-qwen3.8-27b-4-bit-ollama-256",
				"s2-qwen3.8-27b-4-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--qwen3.8-27b--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-256",
							"score": {
								"score": 83,
								"low": 80,
								"high": 83,
								"ratio": 0.8264382077590433
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-llama-256",
							"score": {
								"score": 80,
								"low": 77,
								"high": 81,
								"ratio": 0.7998384304864491
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-256",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-256",
							"score": {
								"score": 78,
								"low": 77,
								"high": 78,
								"ratio": 0.7772158716859178
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-llama-256",
							"score": {
								"score": 75,
								"low": 72,
								"high": 76,
								"ratio": 0.7515111616494106
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-256",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-mlx-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-ollama-256",
							"score": {
								"score": 82,
								"low": 81,
								"high": 82,
								"ratio": 0.8189189098046985
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-4-bit-llama-256",
							"score": {
								"score": 82,
								"low": 81,
								"high": 82,
								"ratio": 0.8188693210495047
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-4-bit-uzu-256",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "qwen3.8-27b",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-qwen3.8-27b-8-bit-llama-2048",
				"s2-qwen3.8-27b-8-bit-magnitude-2048",
				"s2-qwen3.8-27b-8-bit-mlx-2048",
				"s2-qwen3.8-27b-8-bit-ollama-2048",
				"s2-qwen3.8-27b-8-bit-uzu-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-2048",
							"score": {
								"score": 96,
								"low": 95,
								"high": 99,
								"ratio": 0.9582661004257894
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-2048",
							"score": {
								"score": 96,
								"low": 90,
								"high": 97,
								"ratio": 0.9563519210706091
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-llama-2048",
							"score": {
								"score": 92,
								"low": 76,
								"high": 95,
								"ratio": 0.9221982718745174
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-llama-2048",
							"score": {
								"score": 98,
								"low": 78,
								"high": 98,
								"ratio": 0.9784462214690463
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-2048",
							"score": {
								"score": 97,
								"low": 96,
								"high": 97,
								"ratio": 0.9681194725414394
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9993111633715851
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-2048",
							"score": {
								"score": 91,
								"low": 91,
								"high": 91,
								"ratio": 0.9089278675375683
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-2048",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "qwen3.8-27b",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-qwen3.8-27b-8-bit-llama-256",
				"s2-qwen3.8-27b-8-bit-magnitude-256",
				"s2-qwen3.8-27b-8-bit-mlx-256",
				"s2-qwen3.8-27b-8-bit-ollama-256",
				"s2-qwen3.8-27b-8-bit-uzu-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--qwen3.8-27b--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 95,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-llama-256",
							"score": {
								"score": 97,
								"low": 97,
								"high": 99,
								"ratio": 0.9740087431580935
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-256",
							"score": {
								"score": 95,
								"low": 95,
								"high": 96,
								"ratio": 0.954135786260776
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-256",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-llama-256",
							"score": {
								"score": 98,
								"low": 98,
								"high": 98,
								"ratio": 0.9780856555865775
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-256",
							"score": {
								"score": 97,
								"low": 97,
								"high": 97,
								"ratio": 0.9667597323921403
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-256",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9996999816189819
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-qwen3.8-27b-8-bit-mlx-256",
							"score": {
								"score": 98,
								"low": 98,
								"high": 98,
								"ratio": 0.9813808016876415
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						},
						{
							"engine_id": "uzu",
							"state": "unsupported",
							"experiment_id": "s2-qwen3.8-27b-8-bit-uzu-256",
							"score": null,
							"note": "uzu 0.6.17 renders its own Qwen3.8 chat template: 67 input tokens for the 27-token canonical check prompt, so it can't run the frozen prompts (measured in qualification)."
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-2048-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "ternary-bonsai-2-27b",
			"workload": "archive-2048-128",
			"quantClass": "2-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-bonsai2-27b-2-bit-llama-2048",
				"s2-bonsai2-27b-2-bit-magnitude-2048",
				"s2-bonsai2-27b-2-bit-mlx-2048",
				"s2-bonsai2-27b-2-bit-ollama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 97,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-llama-2048",
							"score": {
								"score": 97,
								"low": 94,
								"high": 100,
								"ratio": 0.9688394684844961
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-mlx-2048",
							"score": {
								"score": 76,
								"low": 73,
								"high": 79,
								"ratio": 0.762485170935795
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-mlx-2048",
							"score": {
								"score": 97,
								"low": 91,
								"high": 98,
								"ratio": 0.9724813851260428
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-llama-2048",
							"score": {
								"score": 96,
								"low": 95,
								"high": 96,
								"ratio": 0.9581053198169911
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-ollama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 0.9998167396284919
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-mlx-2048",
							"score": {
								"score": 56,
								"low": 56,
								"high": 56,
								"ratio": 0.5615212921960552
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-2048",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						}
					]
				}
			}
		},
		{
			"id": "glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-256-128",
			"device": "glenn-m5-max-64gb",
			"cohort": "apple-silicon-mac",
			"platform": "desktop",
			"model": "ternary-bonsai-2-27b",
			"workload": "archive-256-128",
			"quantClass": "2-bit",
			"os": "macOS 27",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s2-bonsai2-27b-2-bit-llama-256",
				"s2-bonsai2-27b-2-bit-magnitude-256",
				"s2-bonsai2-27b-2-bit-mlx-256",
				"s2-bonsai2-27b-2-bit-ollama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-m5-max-64gb--macos-27--ternary-bonsai-2-27b--2-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 0.9987988286241085
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-llama-256",
							"score": {
								"score": 98,
								"low": 93,
								"high": 98,
								"ratio": 0.9768101282512758
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-mlx-256",
							"score": {
								"score": 90,
								"low": 85,
								"high": 90,
								"ratio": 0.9012469032091049
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-mlx-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 0.9964008125772832
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-llama-256",
							"score": {
								"score": 95,
								"low": 90,
								"high": 96,
								"ratio": 0.9543587705352411
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						}
					]
				},
				"memory": {
					"counted": true,
					"engines": [
						{
							"engine_id": "ollama",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-ollama-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-llama-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 100,
								"ratio": 0.9998798464219734
							},
							"note": null
						},
						{
							"engine_id": "mlx-lm",
							"state": "scored",
							"experiment_id": "s2-bonsai2-27b-2-bit-mlx-256",
							"score": {
								"score": 82,
								"low": 82,
								"high": 82,
								"ratio": 0.8226098133699478
							},
							"note": null
						},
						{
							"engine_id": "magnitude",
							"state": "unsupported",
							"experiment_id": "s2-bonsai2-27b-2-bit-magnitude-256",
							"score": null,
							"note": "Magnitude 0.2.17 reports this GGUF as Unsupported (magnitude catalog inspect)"
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
				"s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-2048",
				"s4-oneplus13-lfm2.5-1.2b-4-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 102,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-2048",
							"score": {
								"score": 99,
								"low": 97,
								"high": 106,
								"ratio": 0.9891389864333275
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-256",
				"s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-256",
				"s4-oneplus13-lfm2.5-1.2b-4-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-256",
							"score": {
								"score": 83,
								"low": 83,
								"high": 84,
								"ratio": 0.8326343381389253
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-1.2b-8-bit-litert-2048",
				"s4-oneplus13-lfm2.5-1.2b-8-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-2048",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-2048",
							"score": {
								"score": 87,
								"low": 80,
								"high": 91,
								"ratio": 0.8684832584357222
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-1.2b-8-bit-litert-256",
				"s4-oneplus13-lfm2.5-1.2b-8-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-256",
							"score": {
								"score": 86,
								"low": 86,
								"high": 86,
								"ratio": 0.8631226553967425
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-litert-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-1.2b-bf16-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-1.2b-bf16-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-256",
							"score": {
								"score": 100,
								"low": 100,
								"high": 100,
								"ratio": 1
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-2.6b-4-bit-litert-2048",
				"s4-oneplus13-lfm2.5-2.6b-4-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-2048",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 156,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-2.6b-4-bit-litert-256",
				"s4-oneplus13-lfm2.5-2.6b-4-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 107,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-litert-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Its output check failed: 8/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-2.6b-8-bit-litert-2048",
				"s4-oneplus13-lfm2.5-2.6b-8-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-2.6b-8-bit-litert-256",
				"s4-oneplus13-lfm2.5-2.6b-8-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 97,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 7/9 of the checks the model passes at BF16; needs 8 and the mandatory ones (needle-start, continuation-128). Left out, failed by every BF16 configuration: color, sort, needle-end."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-2048",
				"s4-oneplus13-lfm2.5-350m-4-bit-litert-2048",
				"s4-oneplus13-lfm2.5-350m-4-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 98,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-256",
				"s4-oneplus13-lfm2.5-350m-4-bit-litert-256",
				"s4-oneplus13-lfm2.5-350m-4-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-256",
							"score": {
								"score": 100,
								"low": 95,
								"high": 104,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-350m-8-bit-litert-2048",
				"s4-oneplus13-lfm2.5-350m-8-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-2048",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 102,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-2048",
							"score": {
								"score": 76,
								"low": 74,
								"high": 77,
								"ratio": 0.7642794376098417
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-350m-8-bit-litert-256",
				"s4-oneplus13-lfm2.5-350m-8-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 102,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-256",
							"score": {
								"score": 64,
								"low": 59,
								"high": 64,
								"ratio": 0.6355862251130848
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-litert-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-350m--bf16--archive-2048-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-350m-bf16-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-350m--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-2048",
							"score": {
								"score": 100,
								"low": 99,
								"high": 100,
								"ratio": 1
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-oneplus-13--android-16--lfm2.5-350m--bf16--archive-256-128",
			"device": "glenn-oneplus-13",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-oneplus13-lfm2.5-350m-bf16-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-oneplus-13--android-16--lfm2.5-350m--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-256",
							"score": {
								"score": 100,
								"low": 99,
								"high": 101,
								"ratio": 1
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-oneplus13-lfm2.5-350m-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
				"s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-2048",
				"s4-pixel8-lfm2.5-1.2b-4-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Partial: 2 of 6 planned trials were valid."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-2048",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-256",
				"s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-256",
				"s4-pixel8-lfm2.5-1.2b-4-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": {
								"score": 100,
								"low": 93,
								"high": 102,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-256",
							"score": {
								"score": 78,
								"low": 61,
								"high": 84,
								"ratio": 0.7817298899504935
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-et-xnnpack-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-litert-cpu-256",
							"score": null,
							"note": "Its output check failed: 9/12 of the checks the model passes at BF16; needs 11 and the mandatory ones (needle-start, needle-end, continuation-128). Left out, failed by every BF16 configuration: none."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-1.2b-8-bit-litert-2048",
				"s4-pixel8-lfm2.5-1.2b-8-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-2048",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-2048",
							"score": {
								"score": 100,
								"low": 96,
								"high": 103,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-2048",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-2048",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-1.2b-8-bit-litert-256",
				"s4-pixel8-lfm2.5-1.2b-8-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-256",
							"score": {
								"score": 100,
								"low": 91,
								"high": 103,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-256",
							"score": {
								"score": 87,
								"low": 74,
								"high": 93,
								"ratio": 0.8695437866257736
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-litert-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-1.2b-bf16-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-1.2b-instruct",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-1.2b-bf16-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-1.2b-instruct--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-1.2b-bf16-llama-256",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-2.6b-4-bit-litert-2048",
				"s4-pixel8-lfm2.5-2.6b-4-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-2048",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-2048",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-2048",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-2.6b-4-bit-litert-256",
				"s4-pixel8-lfm2.5-2.6b-4-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-2.6b--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-256",
							"score": {
								"score": 100,
								"low": 98,
								"high": 102,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-litert-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-4-bit-llama-256",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-2.6b-8-bit-litert-2048",
				"s4-pixel8-lfm2.5-2.6b-8-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "No trial was valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "No trial was valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-2048",
							"score": null,
							"note": "No trial was valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-2.6b",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-2.6b-8-bit-litert-256",
				"s4-pixel8-lfm2.5-2.6b-8-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-2.6b--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-litert-256",
							"score": null,
							"note": "Partial: 3 of 6 planned trials were valid."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-2.6b-8-bit-llama-256",
							"score": null,
							"note": "Partial: 4 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-2048",
				"s4-pixel8-lfm2.5-350m-4-bit-litert-2048",
				"s4-pixel8-lfm2.5-350m-4-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-2048",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-2048",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-2048",
							"score": null,
							"note": "Partial: 5 of 6 planned trials were valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "4-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-256",
				"s4-pixel8-lfm2.5-350m-4-bit-litert-256",
				"s4-pixel8-lfm2.5-350m-4-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-350m--4-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-256",
							"score": {
								"score": 100,
								"low": 72,
								"high": 114,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "executorch",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-et-xnnpack-256",
							"score": null,
							"note": "Its output check failed: 5/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "litert-lm",
							"state": "ineligible",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-litert-256",
							"score": null,
							"note": "Its output check failed: 6/8 of the checks the model passes at BF16; needs 7 and the mandatory ones (needle-start, needle-end). Left out, failed by every BF16 configuration: arithmetic, sort, word, continuation-128."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-4-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-350m-8-bit-litert-2048",
				"s4-pixel8-lfm2.5-350m-8-bit-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-2048",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-2048",
							"score": {
								"score": 100,
								"low": 91,
								"high": 101,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-2048",
							"score": {
								"score": 81,
								"low": 78,
								"high": 89,
								"ratio": 0.8070823710546575
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-2048",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-2048",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "8-bit",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-350m-8-bit-litert-256",
				"s4-pixel8-lfm2.5-350m-8-bit-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-350m--8-bit--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-256",
							"score": null,
							"note": "No time to first token or peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": true,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-256",
							"score": {
								"score": 100,
								"low": 93,
								"high": 108,
								"ratio": 1
							},
							"note": null
						},
						{
							"engine_id": "litert-lm",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-256",
							"score": {
								"score": 93,
								"low": 84,
								"high": 97,
								"ratio": 0.9291644349223354
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "litert-lm",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-litert-256",
							"score": null,
							"note": "No peak memory measured."
						},
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-8-bit-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-350m--bf16--archive-2048-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-2048-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-350m-bf16-llama-2048"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-350m--bf16--archive-2048-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "failed",
							"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-2048",
							"score": null,
							"note": "No trial was valid."
						}
					]
				}
			}
		},
		{
			"id": "glenn-pixel-8--android-16--lfm2.5-350m--bf16--archive-256-128",
			"device": "glenn-pixel-8",
			"cohort": "android-phone",
			"platform": "mobile",
			"model": "lfm2.5-350m",
			"workload": "archive-256-128",
			"quantClass": "bf16",
			"os": "Android 16",
			"power": "external",
			"methodology": "0.6",
			"experiment_ids": [
				"s4-pixel8-lfm2.5-350m-bf16-llama-256"
			],
			"page_url": "https://www.xybrid.ai/leaderboard/comparisons/glenn-pixel-8--android-16--lfm2.5-350m--bf16--archive-256-128",
			"outcomes": {
				"balanced": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				},
				"speed": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "scored",
							"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-256",
							"score": {
								"score": 100,
								"low": 86,
								"high": 102,
								"ratio": 1
							},
							"note": null
						}
					]
				},
				"memory": {
					"counted": false,
					"engines": [
						{
							"engine_id": "llama.cpp",
							"state": "missing",
							"experiment_id": "s4-pixel8-lfm2.5-350m-bf16-llama-256",
							"score": null,
							"note": "No peak memory measured."
						}
					]
				}
			}
		}
	],
	"rankings": [
		{
			"platform": "overall",
			"objective": "balanced",
			"status": "uncontested",
			"counted_comparisons": 16,
			"needed": 16,
			"missing_platforms": [],
			"rows": [],
			"unranked": [
				{
					"engine_id": "llama.cpp",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "mlx-lm",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "ollama",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "magnitude",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16. Unsupported in 2."
				},
				{
					"engine_id": "rapid-mlx",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16. Unsupported in 2."
				},
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "zml",
					"reason": "coverage",
					"note": "Results in 0 of 16 comparisons; a rank needs all 16."
				}
			]
		},
		{
			"platform": "overall",
			"objective": "speed",
			"status": "uncontested",
			"counted_comparisons": 16,
			"needed": 16,
			"missing_platforms": [],
			"rows": [],
			"unranked": [
				{
					"engine_id": "llama.cpp",
					"reason": "uncontested",
					"note": "Results in all 16 comparisons; no other engine has completed them yet."
				},
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 9 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 5 of 16 comparisons; a rank needs all 16. Unsupported in 2."
				},
				{
					"engine_id": "mlx-lm",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "ollama",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "magnitude",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16. Unsupported in 2."
				},
				{
					"engine_id": "rapid-mlx",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "zml",
					"reason": "coverage",
					"note": "Results in 0 of 16 comparisons; a rank needs all 16."
				}
			]
		},
		{
			"platform": "overall",
			"objective": "memory",
			"status": "uncontested",
			"counted_comparisons": 16,
			"needed": 16,
			"missing_platforms": [],
			"rows": [],
			"unranked": [
				{
					"engine_id": "llama.cpp",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "mlx-lm",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "ollama",
					"reason": "coverage",
					"note": "Results in 4 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "magnitude",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16. Unsupported in 2."
				},
				{
					"engine_id": "rapid-mlx",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 2 of 16 comparisons; a rank needs all 16. Unsupported in 2."
				},
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 16 comparisons; a rank needs all 16."
				},
				{
					"engine_id": "zml",
					"reason": "coverage",
					"note": "Results in 0 of 16 comparisons; a rank needs all 16."
				}
			]
		},
		{
			"platform": "desktop",
			"objective": "balanced",
			"status": "ranked",
			"counted_comparisons": 4,
			"needed": 4,
			"missing_platforms": [],
			"rows": [
				{
					"rank": 1,
					"engine_id": "ollama",
					"score": {
						"score": 93,
						"low": 89,
						"high": 95,
						"ratio": 0.9269218597499962
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				},
				{
					"rank": 2,
					"engine_id": "mlx-lm",
					"score": {
						"score": 91,
						"low": 90,
						"high": 92,
						"ratio": 0.9083615567687823
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				},
				{
					"rank": 3,
					"engine_id": "llama.cpp",
					"score": {
						"score": 90,
						"low": 84,
						"high": 91,
						"ratio": 0.8985294470094101
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				}
			],
			"unranked": [
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "magnitude",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4. Unsupported in 2."
				},
				{
					"engine_id": "rapid-mlx",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4. Unsupported in 2."
				},
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "zml",
					"reason": "coverage",
					"note": "Results in 0 of 4 comparisons; a rank needs all 4."
				}
			]
		},
		{
			"platform": "desktop",
			"objective": "speed",
			"status": "ranked",
			"counted_comparisons": 4,
			"needed": 4,
			"missing_platforms": [],
			"rows": [
				{
					"rank": 1,
					"engine_id": "mlx-lm",
					"score": {
						"score": 95,
						"low": 95,
						"high": 96,
						"ratio": 0.9541032339524058
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				},
				{
					"rank": 2,
					"engine_id": "ollama",
					"score": {
						"score": 93,
						"low": 91,
						"high": 94,
						"ratio": 0.9335214618545107
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				},
				{
					"rank": 3,
					"engine_id": "llama.cpp",
					"score": {
						"score": 92,
						"low": 84,
						"high": 93,
						"ratio": 0.9212135850579201
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				}
			],
			"unranked": [
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "magnitude",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4. Unsupported in 2."
				},
				{
					"engine_id": "rapid-mlx",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4. Unsupported in 2."
				},
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "zml",
					"reason": "coverage",
					"note": "Results in 0 of 4 comparisons; a rank needs all 4."
				}
			]
		},
		{
			"platform": "desktop",
			"objective": "memory",
			"status": "ranked",
			"counted_comparisons": 4,
			"needed": 4,
			"missing_platforms": [],
			"rows": [
				{
					"rank": 1,
					"engine_id": "ollama",
					"score": {
						"score": 100,
						"low": 99,
						"high": 100,
						"ratio": 1
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				},
				{
					"rank": 2,
					"engine_id": "llama.cpp",
					"score": {
						"score": 100,
						"low": 99,
						"high": 100,
						"ratio": 0.9990552677696528
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				},
				{
					"rank": 3,
					"engine_id": "mlx-lm",
					"score": {
						"score": 85,
						"low": 85,
						"high": 85,
						"ratio": 0.8506940162260257
					},
					"comparisons_entered": 4,
					"unsupported": 0,
					"failed": 0
				}
			],
			"unranked": [
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "magnitude",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4. Unsupported in 2."
				},
				{
					"engine_id": "rapid-mlx",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 2 of 4 comparisons; a rank needs all 4. Unsupported in 2."
				},
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 4 comparisons; a rank needs all 4."
				},
				{
					"engine_id": "zml",
					"reason": "coverage",
					"note": "Results in 0 of 4 comparisons; a rank needs all 4."
				}
			]
		},
		{
			"platform": "mobile",
			"objective": "balanced",
			"status": "uncontested",
			"counted_comparisons": 12,
			"needed": 12,
			"missing_platforms": [],
			"rows": [],
			"unranked": [
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "llama.cpp",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				}
			]
		},
		{
			"platform": "mobile",
			"objective": "speed",
			"status": "uncontested",
			"counted_comparisons": 12,
			"needed": 12,
			"missing_platforms": [],
			"rows": [],
			"unranked": [
				{
					"engine_id": "llama.cpp",
					"reason": "uncontested",
					"note": "Results in all 12 comparisons; no other engine has completed them yet."
				},
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 7 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 3 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				}
			]
		},
		{
			"platform": "mobile",
			"objective": "memory",
			"status": "uncontested",
			"counted_comparisons": 12,
			"needed": 12,
			"missing_platforms": [],
			"rows": [],
			"unranked": [
				{
					"engine_id": "executorch",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "litert-lm",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "llama.cpp",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				},
				{
					"engine_id": "uzu",
					"reason": "coverage",
					"note": "Results in 0 of 12 comparisons; a rank needs all 12."
				}
			]
		}
	]
}
