Rankings
Open data
The record schema, v0.3.0.
Every experiment is one JSON file in this shape, pointing to catalogs of engines, hardware cohorts, devices, models and workloads. A record holds raw measurements only: token counts and durations, never rates, summaries, scores or ranks, which are computed from it (methodology, section 4). The validator checks every key, type and enum against these types, then what the values mean together. Times are UTC ISO 8601, hashes lowercase hex SHA-256, URLs https, and null means not recorded.
Every published experiment, as JSON · What is computed from them
// Schema v0.3.0. The files in data/ and fixtures/ are JSON in exactly these shapes: // validate.ts checks every key, type and enum, and dataset.ts checks what the values mean. // // One normalized record, the experiment: one engine, one model artifact, one device and one // workload, measured over several trials. Engines, cohorts, devices, models and workloads are // catalogs it points to by id. Tracks are views over experiments, so the same experiment can // rank an engine and, later, a model. // // Raw only. A file holds what was set up and what was measured: token counts and durations, // never rates, summaries, scores or ranks. Those are computed from these values when the site // builds, under the methodology version, so a change in how a score is computed never edits a // measurement. // // Times are UTC ISO 8601 ("2026-11-06T09:30:00Z"), hashes lowercase hex SHA-256, URLs https. // null means "not recorded" and is never read as zero. // ---------------------------------------------------------------- catalogs /** Every catalog file: data/{engines,cohorts,devices,models,workloads,baskets}.json and fixtures/ alike. */ export interface CatalogFile<T> { schema_version: string; entries: T[]; } /** An engine. `affiliation: 'xybrid'` marks an engine Xybrid builds: every page showing it discloses that. */ export interface EngineEntry { id: string; name: string; maintainer: string; license: string; open_source: boolean; affiliation: 'xybrid' | 'none'; homepage: string; /** True only for invented entries in fixtures/; a measured experiment can never reference one. */ fixture: boolean; } /** Where a device sits in the leaderboard's platform views. */ export type Platform = 'desktop' | 'mobile' | 'embedded'; /** * A hardware cohort: devices of one platform and a similar hardware class, such as * "Apple silicon laptops" or "Android flagships". Experiments are only compared on the same * device; cohorts group devices for the platform views and the hardware filter. */ export interface CohortEntry { id: string; name: string; platform: Platform; description: string; fixture: boolean; } export type FormFactor = 'phone' | 'tablet' | 'laptop' | 'desktop' | 'single-board' | 'other'; export interface DeviceEntry { id: string; name: string; vendor: string; model: string; /** Points to cohorts.json. A device and a cohort never share an id: the hardware filter takes either. */ cohort: string; soc: string; cpu: string; gpu: string | null; npu: string | null; ram_bytes: number; form_factor: FormFactor; fixture: boolean; } /** A base model: the checkpoint every artifact in a comparison is derived from. */ export interface ModelEntry { id: string; name: string; family: string; parameters: number; upstream: { repo: string; revision: string }; license: string; fixture: boolean; } /** What a workload asks the engine to do. v0.3 measures one kind; each new kind brings its own trial shape. */ export type WorkloadKind = 'text-generation'; export interface Sampling { temperature: number; top_p: number | null; top_k: number | null; seed: number | null; greedy: boolean; } export type Measurement = 'streaming' | 'separate-phases'; /** * A workload fixes everything two experiments must share to be compared: the prompt, the * output length, the context, the batch, the sampling and how phases are timed. Every * experiment on a workload repeats these values, and validation checks that they match. */ export interface WorkloadEntry { id: string; name: string; description: string; kind: WorkloadKind; prompt_tokens: number; /** The SHA-256 of the prompt's text, before any chat template. */ prompt_sha256: string; prompt_construction: string; output_tokens: number; context_length: number; batch_size: number; sampling: Sampling; /** streaming: one request, timed from its token stream. separate-phases: prefill and decode timed apart. */ measurement: Measurement; fixture: boolean; } /** A device in a basket, with the OS it is ranked under ("macOS 27", as comparisons name it). */ export interface BasketDevice { id: string; os: string; } /** * A platform's standard basket for Top Engines (METHODOLOGY.md, section 10, "Top Engines"): a * full grid of devices, models, quantization classes and workloads that every ranked engine * must complete. One per platform under data/, one per platform under fixtures/; the fixture * basket applies when fixtures are loaded. Devices are listed by id with their OS, so a newly * catalogued device or OS version changes no ranking until it is added on purpose. */ export interface BasketEntry { id: string; platform: Platform; devices: BasketDevice[]; models: string[]; /** Quantization classes, as experiments record them ("8-bit"). */ quant_classes: string[]; workloads: string[]; fixture: boolean; } // ---------------------------------------------------------------- experiments export type ProvenanceKind = 'measured' | 'fixture'; export type PublicationStatus = 'draft' | 'published' | 'withdrawn'; export type ExperimentStatus = 'complete' | 'partial' | 'failed' | 'unsupported'; export type TimingSource = 'engine' | 'harness'; export interface Provenance { kind: ProvenanceKind; session_id: string; /** FIXTURE_SENTINEL when `kind` is fixture, null otherwise. */ fixture_notice: string | null; operator: { name: string; organization: string; affiliation: 'xybrid' | 'engine-maintainer' | 'independent' }; run_at: string; harness: { name: string; version: string; commit: string | null }; } export interface ChangelogEntry { at: string; note: string; } export interface Publication { status: PublicationStatus; published_at: string | null; reviewed_by: string[]; changelog: ChangelogEntry[]; } export interface ExperimentDevice { /** Points to devices.json. */ id: string; os: { name: string; version: string; build: string | null; kernel: string | null }; /** On Android, low_power_mode is battery saver, and performance_mode the phone's wakefulness (awake, asleep, dozing). */ power: { source: 'mains' | 'battery' | 'usb'; low_power_mode: boolean | null; performance_mode: string | null }; } export interface ExperimentEngine { /** Points to engines.json. */ id: string; version: string; build: { commit: string | null; binary_sha256: string | null; source: 'release' | 'source' | 'package' | 'private-build'; flags: string[]; dependencies: Record<string, string>; }; backend: string; threads: number | null; /** The settings in effect, as the engine reported them. */ runtime_flags: Record<string, string | number | boolean>; /** How the harness talks to the engine: "cli", "http-server", "python", "swift", ... */ interface: string; } export interface ArtifactFile { path: string; sha256: string; size_bytes: number; } /** The exact weights an experiment loaded. Field names follow the registry's model variants. */ export interface Artifact { format: string; repo: string; revision: string; files: ArtifactFile[]; quantization: { /** The engine's own name for it: "Q4_K_M", "mlx-4bit-g64". */ scheme: string; /** The comparable class: "4-bit", "8-bit", "bf16". Experiments in different classes are never compared. */ class: string; weight_bits: number | null; group_size: number | null; kv_cache: string; }; conversion: { method: 'upstream' | 'converted'; tool: string | null; tool_version: string | null; command: string | null; publisher: string; }; } export interface ExperimentModel { /** Points to models.json. */ id: string; artifact: Artifact; } /** The workload as it ran: its catalog entry's values, plus how the trials were scheduled. */ export interface ExperimentWorkload { /** Points to workloads.json. */ id: string; kind: WorkloadKind; prompt: { tokens: number; source: string; sha256: string; template: string | null; construction: string; token_ids_sha256: string }; output_tokens: number; stop_on_eos: boolean; context_length: number; batch_size: number; sampling: Sampling; measurement: Measurement; warmup_runs: number; trials_planned: number; cooldown_s: number; } export interface ThermalReading { celsius: number | null; /** The OS's own label, where it has one: "nominal", "fair", "serious", "critical". */ state: string | null; } export interface Conditions { ambient_c: number | null; thermal: { start: ThermalReading | null; end: ThermalReading | null; /** The rule a trial waited for before it started, e.g. "SoC at or below 50 °C". */ gate: string | null; throttling_detected: boolean | null; }; } /** A timed phase: its tokens and how long they took. Tokens per second is computed, never stored. */ export interface Phase { tokens: number; duration_ms: number; timing_source: TimingSource; } /** One attempted trial. Invalid trials stay in the record with their reason. */ export interface Trial { index: number; valid: boolean; invalid_reason: string | null; ttft_ms: number | null; prefill: Phase | null; decode: Phase | null; peak_memory_bytes: number | null; memory_method: string | null; thermal: { start: ThermalReading | null; end: ThermalReading | null }; output_sha256: string | null; error: string | null; } export interface Quality { method: string | null; status: 'passed' | 'failed' | 'not-run'; reference: string | null; reference_sha256: string | null; threshold: number | null; items: { id: string; passed: boolean; output_sha256: string }[]; detail: string | null; } export interface ExperimentError { phase: 'probe' | 'load' | 'warmup' | 'trial' | 'unload'; trial_index: number | null; message: string; } export interface Unsupported { reason: string; evidence_url: string | null; } export interface Reproduction { commands: string[]; environment: Record<string, string>; notes: string | null; } export interface RawArtifact { /** raw: the harness's own per-trial output, the evidence a published experiment needs. */ kind: 'raw' | 'log' | 'environment' | 'other'; label: string; url: string; sha256: string; } /** One engine, one artifact, one device and one workload, measured over N trials. */ export interface ExperimentRecord { schema_version: string; id: string; methodology_version: string; provenance: Provenance; publication: Publication; device: ExperimentDevice; engine: ExperimentEngine; model: ExperimentModel; workload: ExperimentWorkload; conditions: Conditions; trials: Trial[]; quality: Quality; status: ExperimentStatus; errors: ExperimentError[]; unsupported: Unsupported | null; caveats: string[]; reproduction: Reproduction; artifacts: RawArtifact[]; }