Diversidad y barridos: dos veredictos en vez de uno

"""Opening diversity: how many different games the model is willing to play.

``docs/spec/02`` lists it in the evaluation table -- "entropy of the openings played in 200 games
of its own" -- and it has been ``n/a`` in every report so far. M4 needs it, because the claim the
masters fine-tune makes is precisely a trade: *the Elo goes up and the diversity goes down*. A
number that only measures the first half of that sentence cannot check it.

Two numbers, because they answer different questions and fail in different ways.

**The line entropy** plays ``games`` openings of the model against itself and measures the
entropy of the distribution of distinct opening lines. It is the metric the spec asks for and it
depends on the sampling: read at the deterministic setting the demo plays at, a decoder produces
one single line and the entropy is exactly zero, which is true but says more about the sampler
than about the weights. So it is reported together with the temperature it was read at, like
every other number in this project (D-047).

**The first-move entropy** is analytic: the entropy of the model's own distribution over the
twenty legal first moves, with no sampling at all. It is a property of the weights, it has no
variance between runs, and it is the one to compare across stages. A model that has been
fine-tuned onto a narrow repertoire collapses it whatever the temperature.

Self-play is batched: every game advances one ply at a time, all of them in one forward pass, so
200 games of 12 plies are 12 passes of batch 200 rather than 2 400 passes of batch 1.
"""

from __future__ import annotations

import math
from collections import Counter
from typing import TYPE_CHECKING

import chess
import torch
from pydantic import BaseModel, ConfigDict, Field

from rukh.eval.legality import header
from rukh.infer.sampler import SampleConfig, legal_token_ids, prompt_ids

if TYPE_CHECKING:
    from rukh.models import MoveDecoder
    from rukh.tokenize.uci_vocab import UciTokenizer

__all__ = [
    "DiversityResult",
    "first_move_distribution",
    "first_move_entropy",
    "opening_diversity",
    "shannon_entropy",
]

OPENING_PLIES = 12
"""Six moves a side: long enough to have left the book, short enough that two games sharing it
really are the same opening."""


class DiversityResult(BaseModel):
    """What ``opening_diversity`` measured, with the sampling it was measured at."""

    model_config = ConfigDict(extra="forbid")

    games: int
    plies: int
    distinct_lines: int
    entropy_bits: float
    """Entropy of the distribution over opening lines, in bits."""
    max_entropy_bits: float
    """``log2(games)``: what the entropy would be if every game opened differently."""
    normalised: float = Field(ge=0.0, le=1.0)
    """``entropy_bits / max_entropy_bits``: 1 is a different opening every game, 0 is always
    the same one. It is what makes two runs with different ``games`` comparable."""
    first_move_entropy_bits: float
    """Entropy of the model's distribution over the legal first moves; no sampling involved."""
    temperature: float
    top_k: int | None
    top_lines: list[tuple[str, int]]
    """The most played lines and how often, for the report; a collapsed model shows it here."""

def shannon_entropy(counts: dict[str, int] | Counter[str]) -> float:
    """Entropy of a distribution given as counts, in bits. Empty or single-valued gives 0."""
    total = sum(counts.values())
    if total <= 0:
        return 0.0
    bits = -sum((n / total) * math.log2(n / total) for n in counts.values() if n > 0)
    # A single outcome gives `-1 * log2(1)`, which is negative zero and renders as "-0.000".
    return 0.0 if bits == 0 else bits

@torch.no_grad()
def opening_diversity(
    model: MoveDecoder,
    tok: UciTokenizer,
    games: int = 200,
    plies: int = OPENING_PLIES,
    sampling: SampleConfig | None = None,
    header_elo: int = 1800,
    seed: int = 42,
) -> DiversityResult:
    """Play ``games`` openings of the model against itself and measure how varied they are."""
    cfg = sampling or SampleConfig()
    device = next(model.parameters()).device
    generator = torch.Generator(device=device).manual_seed(seed)
    start = header(tok, header_elo, header_elo)

    boards = [chess.Board() for _ in range(games)]
    histories = [list(start) for _ in range(games)]
    lines: list[list[str]] = [[] for _ in range(games)]
    alive = list(range(games))

    for _ in range(plies):
        if not alive:
            break
        batch = [prompt_ids(histories[i], model.cfg.block) for i in alive]
        # Every live game is at the same ply, so the batch is rectangular and needs no padding.
        # That is worth an assertion rather than a pad: this model has *learned* position
        # embeddings, so padding on the left would shift every real token to a position it was
        # never trained at, and padding on the right would leave the last column holding a
        # `<pad>` for the shorter rows. Both read the wrong distribution, silently.
        width = len(batch[0])
        if any(len(ids) != width for ids in batch):
            raise RuntimeError("self-play games fell out of step; the batch cannot be padded")
        padded = torch.tensor(batch, dtype=torch.long, device=device)
        logits = model(padded)[0][:, -1, :].float()

        still: list[int] = []
        for row, index in enumerate(alive):
            board = boards[index]
            legal = legal_token_ids(board, tok)
            if not legal:
                continue
            filtered = _mask(logits[row], legal, cfg)
            probs = torch.softmax(filtered, dim=-1)
            if cfg.temperature == 0:
                token = int(torch.argmax(filtered))
            else:
                token = int(torch.multinomial(probs, 1, generator=generator))
            uci = tok.ids[token]
            move = chess.Move.from_uci(uci)
            board.push(move)
            histories[index].append(token)
            lines[index].append(uci)
            if not board.is_game_over():
                still.append(index)
        alive = still

    counts = Counter(" ".join(line) for line in lines if line)
    entropy = shannon_entropy(counts)
    ceiling = math.log2(games) if games > 1 else 0.0
    return DiversityResult(
        games=games,
        plies=plies,
        distinct_lines=len(counts),
        entropy_bits=entropy,
        max_entropy_bits=ceiling,
        normalised=min(entropy / ceiling, 1.0) if ceiling > 0 else 0.0,
        first_move_entropy_bits=first_move_entropy(model, tok, header_elo),
        temperature=cfg.temperature,
        top_k=cfg.top_k,
        top_lines=counts.most_common(5),
    )

def _mask(logits: torch.Tensor, legal: list[int], cfg: SampleConfig) -> torch.Tensor:
    """The legality mask, the temperature and the top-k of ``SampleConfig``, on one row.

    Always masked, whatever ``cfg.mask_illegal`` says: this measures which *openings* the model
    chooses, and an illegal token is not an opening. Legality has its own metric.
    """
    out = logits.clone()
    keep = torch.zeros_like(out, dtype=torch.bool)
    keep[torch.tensor(legal, dtype=torch.long, device=out.device)] = True
    out = out.masked_fill(~keep, float("-inf"))
    if cfg.temperature > 0:
        out = out / cfg.temperature
    if cfg.top_k is not None:
        k = min(cfg.top_k, len(legal))
        threshold = torch.topk(out, k).values[-1]
        out = out.masked_fill(out < threshold, float("-inf"))
    return out

def _diversity_section(detail: DiversityResult | None) -> list[str]:
    """How varied the model's own openings are, and the two numbers that say it.

    Split in two because they fail differently. The line entropy depends on the sampler -- at the
    near-deterministic setting the stages are compared at, every decoder plays one opening and
    scores 0 -- so it is read at its own temperature and always reported with it. The first-move
    entropy has no sampler in it at all: it is the model's own distribution over the twenty legal
    first moves, so it is the one to compare across stages.
    """
    if detail is None:
        return []
    top_k = "none" if detail.top_k is None else str(detail.top_k)
    lines = [
        "## Opening diversity",
        "",
        f"{detail.games} self-play openings of {detail.plies} plies, drawn at temperature "
        f"{detail.temperature} with top-k {top_k}. **Not** the suite's sampling: at the "
        "near-deterministic setting every stage plays one single opening and scores 0, which "
        "measures the sampler and not the weights (D-047).",
        "",
        "| Metric | Value |",
        "|---|---:|",
        f"| Distinct opening lines | {detail.distinct_lines} of {detail.games} |",
        f"| Line entropy | {detail.entropy_bits:.3f} bits of {detail.max_entropy_bits:.3f} |",
        f"| Normalised | {detail.normalised:.3f} |",
        f"| First-move entropy (no sampling) | {detail.first_move_entropy_bits:.3f} bits |",
        "",
        "Most played lines:",
        "",
        "| Line | Games |",
        "|---|---:|",
    ]
    lines.extend(f"| `{line}` | {count} |" for line, count in detail.top_lines)
    lines.append("")
    return lines

"""Measure one model once per Elo header: does asking it to play weaker actually work?

``GOAL.md`` asks P4 for a monotonic Elo by condition -- 1500 < 2000 < 2400, with intervals -- and
nothing in the harness could answer that, because ``header_elo`` was one number in a config and a
suite run produced one row. This runs the very same suite once per condition, changing that
number and **nothing else**: same opponents, same positions, same puzzles, same seed, same
temperature. Anything that differs between the rows is the header.

Two claims are checked separately and reported separately, because they are not the same claim.
A **monotonic** sweep is one whose point estimates go up with the condition; a **separated** one
is a sweep whose consecutive confidence intervals do not overlap. Point estimates rise by chance
all the time -- that is exactly how D-069 first read a 32-game trial as a win for `<2600>` and
had to take it back at 160 games -- so the sweep prints both and the milestone is judged on the
second.

The cache makes this much cheaper than it sounds: a game is keyed by the weights, the config and
the header, so re-running a sweep after adding one condition only plays the new one.
"""

from __future__ import annotations

import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING

from pydantic import BaseModel, ConfigDict

from rukh import paths
from rukh.eval.suite import EvalConfig, run_suite

if TYPE_CHECKING:
    from rukh.eval.suite import SuiteResult

log = logging.getLogger(__name__)

__all__ = ["SweepResult", "SweepRow", "elo_summary", "run_sweep", "sweep_row", "write_sweep"]

WEB_FILE = "artifacts/web/elo-conditioning.json"

class SweepRow(BaseModel):
    """One condition: what the model does when it is told to play at this rating."""

    model_config = ConfigDict(extra="forbid")

    header_elo: int
    elo: float | None = None
    elo_ci: list[float] | None = None
    elo_lower: float | None = None
    elo_upper: float | None = None
    score: float | None = None
    """Points per game over the whole ladder; the raw number the Elo is fitted from."""
    legality: float | None = None
    """Asked to play badly, does it play badly or does it play *illegally*? Not the same thing,
    and the difference is the whole question of whether the header is understood."""
    top1: float | None = None
    puzzles: float | None = None
    first_move_entropy: float | None = None
    stage: str

def _ci_text(row: SweepRow) -> str:
    if row.elo_ci:
        return f"{row.elo_ci[0]:.0f}-{row.elo_ci[1]:.0f}"
    if row.elo_lower is not None:
        return f"> {row.elo_lower:.0f}"
    if row.elo_upper is not None:
        return f"< {row.elo_upper:.0f}"
    return "n/a"


def sweep_row(header_elo: int, result: SuiteResult) -> SweepRow:
    """Pull the columns of one condition out of a full suite result."""
    elo = result.elo
    return SweepRow(
        header_elo=header_elo,
        elo=elo.elo if elo else None,
        elo_ci=[elo.ci_low, elo.ci_high] if elo and not elo.separated else None,
        elo_lower=elo.elo_lower if elo else None,
        elo_upper=elo.elo_upper if elo else None,
        score=elo.score if elo else None,
        legality=result.legality_argmax.rate if result.legality_argmax else None,
        top1=result.accuracy.top1 if result.accuracy else None,
        puzzles=result.puzzles.rate if result.puzzles else None,
        first_move_entropy=(
            result.diversity_detail.first_move_entropy_bits if result.diversity_detail else None
        ),
        stage=result.stage,
    )

def run_sweep(
    ckpt: Path | str,
    cfg: EvalConfig,
    elos: list[int],
    suite: str = "full",
    use_cache: bool = True,
    device: str | None = None,
    stage: str | None = None,
) -> SweepResult:
    """Run the suite once per condition and collect the rows into one verdict."""
    if len(elos) < 2:
        raise ValueError("a sweep needs at least two conditions to compare")
    if sorted(elos) != elos:
        raise ValueError(f"conditions must be given in increasing order, got {elos}")
    base_stage = stage or cfg.stage or Path(ckpt).parent.name
    rows: list[SweepRow] = []
    last: SuiteResult | None = None
    for header_elo in elos:
        name = f"{base_stage}@{header_elo}"
        log.info("condition %s of %s: %s", len(rows) + 1, len(elos), name)
        per_condition = cfg.model_copy(update={"header_elo": header_elo, "stage": name})
        result, _ = run_suite(ckpt, per_condition, suite=suite, use_cache=use_cache, device=device)
        rows.append(sweep_row(header_elo, result))
        last = result
    assert last is not None
    measured = [row.elo for row in rows if row.elo is not None]
    return SweepResult(
        stage=base_stage,
        checkpoint=Path(ckpt).as_posix(),
        params=last.params,
        date=last.date,
        rows=rows,
        monotonic=is_monotonic(rows),
        separated=is_separated(rows),
        span=(max(measured) - min(measured)) if len(measured) >= 2 else None,
    )

def write_sweep(result: SweepResult, out_dir: Path | str, web: Path | str | None = None) -> Path:
    """Write the sweep's table and JSON, and the file the course reads."""
    directory = Path(out_dir) / f"{result.stage}-elo-sweep"
    directory.mkdir(parents=True, exist_ok=True)
    markdown = directory / "report.md"
    markdown.write_text(_render(result), encoding="utf-8", newline="\n")
    payload = json.loads(result.model_dump_json())
    (directory / "results.json").write_text(
        json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline="\n"
    )
    if web is not None:
        target = paths.resolve(str(web))
        target.parent.mkdir(parents=True, exist_ok=True)
        target.write_text(
            json.dumps(payload, indent=2, ensure_ascii=False) + "\n",
            encoding="utf-8",
            newline="\n",
        )
    return markdown


def _render(result: SweepResult) -> str:
    verdict = "monotonic" if result.monotonic else "**not** monotonic"
    separation = "separated" if result.separated else "**not** separated"
    span = "n/a" if result.span is None else f"{result.span:.0f} Elo"
    return "\n".join(
        [
            f"# Elo by condition of `{result.stage}`",
            "",
            f"- Checkpoint: `{result.checkpoint}`",
            f"- Parameters: {result.params:,}",
            f"- Date: {result.date}",
            "",
            "Every row is the same suite with one number changed: the `<wXXXX> <bXXXX>` header "
            "the model is prompted with. Same opponents, same positions, same puzzles, same "
            "seed, same temperature.",
            "",
            result.table(),
            "",
            "## Verdict",
            "",
            f"- Point estimates: {verdict}.",
            f"- Confidence intervals: {separation}.",
            f"- Span between the weakest and the strongest condition: {span}.",
            "",
            "The two are not the same claim. Point estimates rise by chance often enough that a "
            "monotonic row of numbers is not evidence on its own; the acceptance bar of `GOAL.md` "
            "is read on the intervals.",
            "",
        ]
    )

def elo_summary(rows: list[SweepRow]) -> str:
    """One line per condition for the CLI."""
    lines = [
        f"<w{row.header_elo:04d}>: "
        + ("n/a" if row.elo is None else f"{row.elo:.0f} ({_ci_text(row)})")
        for row in rows
    ]
    return "\n".join(lines)

def sample_positions(
    games: Path,
    n: int,
    tok: UciTokenizer,
    seed: int = 0,
    pool: int = 50_000,
    block: int = 200,
    header_elo: int | None = None,
) -> list[Position]:
    """Sample ``n`` prefixes from a UCI games parquet, one per game, at a random ply.

    Only the first ``pool`` rows are read (a validation month is millions of games and the
    harness never needs more than a few tens of thousands); which of them are used, and at which
    ply, is decided by ``seed``.

    ``header_elo`` replaces the ratings the game really had. It is off by default, because the
    honest way to score next-move accuracy is to show the model the header the game actually
    carried. It is on for the Elo sweep, and for the same reason the puzzles need it: every
    position is prompted with the *condition* being tested, so "asked to play at 1200, does it
    play worse or does it play illegally?" becomes a question the table can answer. Without it
    those two columns are identical in every row -- true, and indistinguishable from evidence
    that the condition does nothing.
    """

def start_history(
    tok: UciTokenizer,
    item: PuzzleItem,
    header_elo: int = HEADER_ELO,
    force_header: bool = False,
) -> list[int]:
    """The ids the model sees before its first move: header, then the real game if there is one.

    The header is the ``<bos> <wXXXX> <bXXXX>`` of ``rukh.infer.sampler.prompt_ids``, with the
    players' own ratings when the parquet carries them; ``header_elo`` stands in otherwise.

    ``force_header`` overrides the real ratings, and exists for exactly one measurement: the Elo
    sweep of M4, which asks the same model to play at several strengths and needs *every* column
    to answer that question. Almost all puzzles carry their players' ratings, so without this the
    puzzle rate would be identical in every row of the sweep and would look like evidence that
    the condition does nothing.
    """
    if force_header:
        white = black = header_elo
    else:
        white = item.white_elo if item.white_elo is not None else header_elo
        black = item.black_elo if item.black_elo is not None else header_elo
    history = header(tok, white, black)
    history.extend(tok.vocab.get(uci, tok.unk_id) for uci in item.prefix)
    return history

    items: Sequence[PuzzleItem],
    cache: EvalCache | None = None,
    header_elo: int = HEADER_ELO,
    force_header: bool = False,
) -> PuzzleResult:
    """Attempt every puzzle, reusing the cached attempts of a previous run of the same weights."""
    attempts: list[PuzzleAttempt] = []
    for item in items:
        # A puzzle whose parquet carries the players' own ratings ignores ``header_elo``, so its
        # key stays as it was; one that falls back to the header is a different attempt per
        # header and must not read back the 1800 run's answer.
        uses_header = force_header or item.white_elo is None or item.black_elo is None
        key = item.puzzle_id
        if uses_header and header_elo != HEADER_ELO:
            key = f"{key}:e{header_elo}"
        cached = cache.get(SUITE, key) if cache is not None else None
        if cached is not None:
            attempt = PuzzleAttempt.model_validate(cached)
            # An attempt played with the other prompt answers a different question.
            if attempt.prompt_style == item.prompt_style:
                attempts.append(attempt)
                continue
        attempt = solve_puzzle(source, tok, item, header_elo=header_elo, force_header=force_header)
        if cache is not None:
            cache.put(SUITE, key, attempt.model_dump())
        attempts.append(attempt)
    return summarize(attempts)

    def item_id(self) -> str:
        """Cache key. The header is only in it when it is not the default.

        Without the header a run at ``<2600>`` would silently read back the games played at
        ``<1800>``: same rung, same index, completely different game. Leaving the default out
        keeps every game cached before this existed addressable.
        """
        suffix = "" if self.header_elo == 1800 else f":e{self.header_elo}"
        return f"{self.rung}:{self.index}{suffix}"

# The four ``Skill Level`` rungs carry **measured** ratings, not the nominal 800/950/1100/1250 they
# were given when the harness was written. Those guesses were wrong by 430 to 580 Elo and dragged
# every stage's fit down by about 350 points: the contradiction that exposed them was a model
# scoring 0.725 against a 1320-rated engine while losing 20-0 to one labelled 1250.
#
# The numbers below come from playing the ladder against itself at the suite's own 0.1 s per move,
# 40 games per pair with colours alternated, anchored on ``uci-1320`` (D-070). The method validates
# on its own control: ``uci-1500`` measured +179 Elo over ``uci-1320`` against a nominal +180.
# ``UCI_Elo`` does compress higher up -- ``uci-1800`` measured +215 over ``uci-1500``, not +300 --
# which is a caveat for the top rungs and not for the range these models play in.
# Ordered by measured strength. Note what that ordering shows: **no rung is below 1320**. The
# four ``skill-*`` opponents were put in the ladder to reach under the engine's ``UCI_Elo`` floor
# and they never did, which is why every stage measured lower than it plays.
DEFAULT_RUNGS: list[EloRung] = [
    EloRung(name="uci-1320", elo=1320, uci_elo=1320),
    EloRung(name="skill-0", elo=1381, skill=0),
    EloRung(name="skill-1", elo=1467, skill=1),
    EloRung(name="uci-1500", elo=1500, uci_elo=1500),
    EloRung(name="skill-2", elo=1589, skill=2),
    EloRung(name="skill-3", elo=1678, skill=3),
    EloRung(name="uci-1800", elo=1800, uci_elo=1800),
    EloRung(name="uci-2000", elo=2000, uci_elo=2000),
]

def play_rung(
    model: MoveDecoder | None,
    tok: UciTokenizer,
    rung: EloRung,
    games: int,
    cfg: SampleConfig,
    move_time: float = MOVE_TIME,
    max_plies: int | None = None,
    cache: EvalCache | None = None,
    header_elo: int = 1800,
    player: Player | None = None,
) -> list[GameRecord]:
    """Play ``games`` games against one rung, alternating colours, reusing cached games.

    A game that ends with ``*`` (the context ran out) is adjudicated with the rung's own engine
    before it is scored, so it enters the fit as a win, a loss or a draw on the merits of the
    final position.

    ``player`` replaces the decoder, and it is how M4 puts a general language model on this exact
    ladder -- same opponents, same move time, same colours, same adjudication. A comparison whose
    two halves went through different code is not a comparison. When it is given, ``model`` is
    unused and may be ``None``; the seed still varies per game, but a player that cannot be
    seeded simply ignores it.
    """

    accuracy_positions: int = 10_000
    position_pool: int = 50_000
    puzzles_per_band: int = 2_000
    force_header: bool = False
    """Prompt everything with ``header_elo`` instead of the ratings the game really had.

    Off everywhere except the Elo sweep, where it is what makes the table readable. Validation
    positions carry their own players' ratings and almost every puzzle carries its game's, so
    with it off the legality, the accuracy and the puzzle rate come out **identical in every row**
    of a sweep: correct by their own definition, and indistinguishable from evidence that the
    condition does nothing. The Elo games never had this problem -- they are played from the
    starting position and the header is the only thing that says who is playing."""
    header_elo: int = 1_800
    """The Elo the model is asked to play at (``<wXXXX> <bXXXX>``).

    It was hardcoded to 1800 while the corpus was 1800+ Lichess and the choice cost nothing. With
    over half the corpus at 2400+ (D-064) it is a real knob: asking a model trained on masters to
    imitate an 1800 is asking it for its weaker mode, so the header belongs in the suite config
    and in the cache key rather than in a default argument.
    """
    elo_games: int = 100
    elo_rungs: list[EloRung] = Field(default_factory=lambda: list(DEFAULT_RUNGS))
    elo_move_time: float = 0.1
    elo_max_plies: int | None = None
    bootstrap: int = 1_000
    diversity_games: int = 200
    """Self-play openings for the diversity metric; 0 switches it off."""
    diversity_plies: int = 12
    diversity_temperature: float = 1.0
    """Read at its own temperature, not the suite's. The suite plays near-deterministically so
    that stages are comparable (D-047), and at that setting a decoder plays one single opening
    and the entropy is 0 for every model alike. Diversity only says something where there is a
    choice to make, so it gets the temperature that leaves one."""
    diversity_top_k: int | None = 20
    """And its own top-k, for the same reason and a sharper one: the suite reads at ``top_k: 1``,
    which is argmax whatever the temperature says. Overriding the temperature alone would look
    like a diversity measurement and be a second copy of the deterministic one."""

def eval_sweep_cmd(
    model: Annotated[
        str, typer.Option("--model", help="Checkpoint path or Hub id to evaluate per condition.")
    ],
    elos: Annotated[
        str,
        typer.Option("--elos", help="Increasing Elo headers to ask for, comma separated."),
    ] = "1200,1500,1800,2100,2400",
    suite: Annotated[str, typer.Option("--suite", help="Suite name: full or quick.")] = "full",
    config: Annotated[
        Path | None,
        typer.Option(
            "--config", exists=True, dir_okay=False, readable=True, help="Suite YAML override."
        ),
    ] = None,
    stage: Annotated[str | None, typer.Option("--stage", help="Name of the sweep.")] = None,
    no_cache: Annotated[bool, typer.Option("--no-cache", help="Recompute every game.")] = False,
    device: Annotated[str | None, typer.Option("--device", help="Where to run.")] = None,
    web: Annotated[
        bool, typer.Option("--web/--no-web", help="Also write the JSON the course reads.")
    ] = True,
) -> None:
    """Run the suite once per Elo header and say whether the ratings come out monotonic.

    Every row is the same evaluation with one number changed, so anything that differs between
    them is the condition. The verdict is printed twice on purpose: whether the point estimates
    rise, and whether the confidence intervals actually separate. Only the second is evidence.
    """
    from rukh.eval import load_suite
    from rukh.eval.suite import SUITES, is_hub_id
    from rukh.eval.sweep import WEB_FILE, elo_summary, run_sweep, write_sweep
    from rukh.paths import resolve

    if suite not in SUITES:
        typer.echo(f"error: --suite must be one of {', '.join(SUITES)}", err=True)
        raise typer.Exit(code=2)
    if not Path(model).is_file() and not is_hub_id(model):
        typer.echo(f"error: --model {model!r} is neither a checkpoint nor a Hub id", err=True)
        raise typer.Exit(code=2)
    try:
        wanted = [int(part) for part in elos.split(",") if part.strip()]
    except ValueError as exc:
        typer.echo(f"error: --elos must be a comma-separated list of integers: {exc}", err=True)
        raise typer.Exit(code=2) from exc
    logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(message)s")
    cfg = load_suite(suite, config)
    try:
        result = run_sweep(
            model, cfg, wanted, suite=suite, use_cache=not no_cache, device=device, stage=stage
        )
    except (FileNotFoundError, ValueError) as exc:
        typer.echo(f"error: {exc}", err=True)
        raise typer.Exit(code=1) from exc
    markdown = write_sweep(result, resolve(cfg.out_dir), WEB_FILE if web else None)
    typer.echo(f"stage:      {result.stage} ({result.params:,} parameters)")
    typer.echo(elo_summary(result.rows))
    typer.echo(f"monotonic:  {'yes' if result.monotonic else 'no'} (point estimates)")
    typer.echo(f"separated:  {'yes' if result.separated else 'no'} (confidence intervals)")
    if result.span is not None:
        typer.echo(f"span:       {result.span:.0f} Elo")
    typer.echo(f"report:     {markdown}")

def eval_openings_cmd(
    model: Annotated[str, typer.Option("--model", help="Checkpoint or Hub id to read.")],
    against: Annotated[
        str | None,
        typer.Option("--against", help="A second model to compare the distribution with."),
    ] = None,
    header_elo: Annotated[
        int, typer.Option("--elo", help="Elo header to ask the distribution at.")
    ] = 1800,
    top: Annotated[int, typer.Option("--top", help="How many first moves to print.")] = 8,
    device: Annotated[str | None, typer.Option("--device", help="Where to run.")] = None,
) -> None:
    """Print what the model would open with, as probabilities rather than as sampled games.

    No temperature, no top-k, no seed: the softmax over the twenty legal first moves as the
    weights produce it. That is what makes it the right instrument for an adapter's effect --
    the share it moved is a property of the weights and does not have to be averaged out of
    hundreds of self-play games.
    """
    from rukh.eval.diversity import first_move_distribution, first_move_entropy
    from rukh.eval.suite import resolve_model
    from rukh.tokenize.uci_vocab import UciTokenizer
    from rukh.train import load_model, pick_device

    logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(message)s")
    where = device or pick_device()
    tok = UciTokenizer()

    def read(path: str) -> tuple[dict[str, float], float]:
        loaded, _ = load_model(resolve_model(path), map_location=where)
        loaded = loaded.to(where).eval()
        return first_move_distribution(loaded, tok, header_elo), first_move_entropy(
            loaded, tok, header_elo
        )

    try:
        probs, entropy = read(model)
        other = read(against) if against is not None else None
    except (FileNotFoundError, ValueError) as exc:
        typer.echo(f"error: {exc}", err=True)
        raise typer.Exit(code=1) from exc

    typer.echo(f"header:   <w{header_elo:04d}>")
    typer.echo(f"entropy:  {entropy:.4f} bits of {math.log2(20):.4f}")
    if other is not None:
        typer.echo(f"          {other[1]:.4f} bits for {against}")
    order = sorted(probs, key=lambda move: probs[move], reverse=True)[:top]
    header_row = "move      share" + ("     other     delta" if other is not None else "")
    typer.echo(header_row)
    for move in order:
        line = f"{move:<9s} {probs[move] * 100:6.2f} %"
        if other is not None:
            mine, theirs = probs[move], other[0].get(move, 0.0)
            line += f"  {theirs * 100:6.2f} %  {(theirs - mine) * 100:+6.2f}"
        typer.echo(line)

# The sampling the acceptance bar is read at: near-deterministic (D-047). Temperature is worth
# more than 200 Elo on the same weights, so every comparison between stages has to fix it, and
# this file is that fixture -- it reproduces exactly what `small-greedy` and `medium-greedy` were
# measured with, so a new stage lands in the same table.
#
# `games` is the month whose first 20,000 rows the position pool reads (`.head(pool)`), and the
# first 100,000 games of that month are the held-out validation split (D-062). The pool is
# therefore inside the held-out part even now that the rest of the month trains.
games: data/uci/year=2025/month=02/games.parquet
puzzles: data/puzzles/puzzles.parquet
puzzle_split: test
legality_positions: 1000
accuracy_positions: 1000
position_pool: 20000
puzzles_per_band: 2000
elo_games: 20
elo_move_time: 0.1
bootstrap: 500
temperature: 0.05
top_k: 1
block: 200
seed: 42
header_elo: 1800   # the header every published number was measured at; keep it pinned
out_dir: artifacts/eval
# The single results table, which this file stopped writing to and should not have. Every number
# the project publishes is read at this sampling, so `web_results: null` left the table carrying
# the rows of the `full` suite -- measured before D-070 corrected the Elo ladder, and therefore
# retracted. `/proyecto/` served them.
web_results: artifacts/web/results.json
cache_db: artifacts/eval/cache-greedy.sqlite
track: false

# The suite the Elo sweep of M4 runs once per condition.
#
# Identical to `greedy.yaml` -- the sampling every published number was read at (D-047) -- with
# one change: `force_header`. Validation positions carry their own players' ratings and almost
# every puzzle carries its game's, so with it off the legality, the accuracy and the puzzle rate
# come out identical in every row of the sweep. Correct by their own definition, and
# indistinguishable from evidence that the condition does nothing, which is the one thing this
# table must not be ambiguous about.
#
# Its own cache, for the same reason: these puzzle attempts answer a different question from the
# ones `cache-greedy.sqlite` holds and must not be confused with them.

force_header: true

header_elo: 1800
diversity_games: 200
diversity_plies: 12
diversity_temperature: 1.0
diversity_top_k: 20
out_dir: artifacts/eval
web_results: null
cache_db: artifacts/eval/cache-sweep.sqlite
track: false

# `greedy.yaml` without the Stockfish ladder: everything that does not need an engine.
#
# For the stages where the question is "did this damage the model?" rather than "how strong is
# it?": the style adapters of M4. Legality, next-move accuracy and puzzles all answer the first
# question, cost minutes instead of an hour, and puzzles in particular track playing strength
# closely enough to catch an adapter that broke something (in D-072 they moved 22 -> 37 % across
# a 145-Elo gap).
#
# What it cannot do is replace an Elo. A stage measured with this file has no rating and its card
# says so; it does not borrow one from its base model.

# Same as `greedy.yaml` except the Elo header. Reported alongside the 1800 row, never
# instead of it: 1800 is what the published stages were measured at, and mixing the two
# in one table would compare a model against a different question.

# A copy of `greedy.yaml` with its own cache, to replay games that are already measured
# without overwriting them. Used once, to find out whether CPU load on the machine changes
# what Stockfish plays at 0.1 s per move -- if it does, no Elo number taken under load is
# comparable with a number taken on an idle machine.

# El barrido otra vez, muestreando la distribución en vez de su moda.
#
# Toda la suite juega con `temperature: 0.05, top_k: 1`, que es la moda: el modelo hace siempre su
# jugada más probable. Y la diferencia entre un 1200 y un 2400 no está en la moda -- los dos hacen
# la recaptura obvia -- sino en la cola, en cada cuánto eligen algo malo. Los datos del hito lo
# dicen solos: la entropía de la primera jugada, que lee la distribución entera, es monótona en las
# seis condiciones; los puzles y el Elo, que leen la moda, son planos.
#
# Así que esta tirada cambia una sola cosa respecto de `greedy-sweep.yaml`: muestrea a temperatura
# 1,0 con top-k 20, que es el ajuste "humano" de la demo. Si la brecha entre `<w1200>` y `<w2400>`
# se abre, la cabecera controla la fuerza y lo que faltaba era dejar de muestrear el pico. Si sale
# plana, es un segundo resultado nulo y también dice algo.
#
# NO sustituye a nada publicado: D-047 fija el muestreo casi determinista como el de todos los
# números del proyecto, así que esto es una medición aparte, con su caché aparte, y se publica
# diciendo a qué temperatura se leyó.

def drop_rows(path: Path, stages: list[str]) -> tuple[list[str], list[dict[str, Any]]]:
    """Remove rows from the shared results file; return ``(removed stages, rows that stay)``.

    A measurement can be **retracted**, and when it is, the table has to be able to say so by not
    carrying it any more. It happened once already: D-070 found that four of the eight rungs of
    the Elo ladder had invented ratings, wrong by about five hundred points, which made every Elo
    published before it wrong. The corrected runs were written under new stage names, so the old
    rows stayed in the table and the project page kept serving retracted numbers for a day.

    Deliberately explicit and per stage rather than a filter or a wildcard: dropping a row is
    throwing away a measurement, and it should take saying its name.
    """
    path = Path(path)
    if not path.is_file():
        return [], []
    payload = json.loads(path.read_text(encoding="utf-8"))
    found = payload.get("rows") if isinstance(payload, dict) else payload
    rows = [item for item in found if isinstance(item, dict)] if isinstance(found, list) else []
    wanted = set(stages)
    removed = sorted({str(item.get("stage")) for item in rows if item.get("stage") in wanted})
    kept = [item for item in rows if item.get("stage") not in wanted]
    document = {"updated_at": datetime.now(UTC).isoformat(timespec="seconds"), "rows": kept}
    path.write_text(
        json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline="\n"
    )
    return removed, kept

BENCHMARKS_MARK = "## Resultados"
"""Heading of ``docs/benchmarks.md`` from which everything is rewritten by ``render_benchmarks``."""

BENCHMARKS_COLUMNS: list[tuple[str, str]] = [
    ("Etapa", ""),
    ("Legalidad sin máscara (%)", "---:"),
    ("Top-1 (%)", "---:"),
    ("Top-3 (%)", "---:"),
    ("Puzles 1000-1500 (%)", "---:"),
    ("Puzles 1500-2000 (%)", "---:"),
    ("Puzles 2000+ (%)", "---:"),
    ("Elo estimado (IC 95 %)", ""),
    ("Δcp medio", "---:"),
    ("Entropía 1.ª jugada (bits)", "---:"),
    ("Fecha", ""),
]

def render_benchmarks(rows: list[dict[str, Any]]) -> str:
    """The results table of ``docs/benchmarks.md``, rendered from the measured rows.

    Written rather than typed, and for the reason the whole project keeps repeating: a table
    copied by hand from a report is a second source of truth that starts drifting on the day it
    is written. The encoder's rows are left out -- they share not one column with these -- and so
    is every row that has no measurement at all.
    """
    playing = sorted(
        (row for row in rows if row.get("kind") != "encoder"), key=lambda row: str(row.get("stage"))
    )
    header = "| " + " | ".join(name for name, _ in BENCHMARKS_COLUMNS) + " |"
    rule = "|" + "|".join(align or "---" for _, align in BENCHMARKS_COLUMNS) + "|"
    lines = [BENCHMARKS_MARK, "", header, rule]
    for row in playing:
        lines.append(
            "| "
            + " | ".join(
                [
                    f"`{row.get('stage')}`",
                    _percent(row.get("legality")),
                    _percent(row.get("top1")),
                    _percent(row.get("top3")),
                    _band(row, "puzzles", "1000-1500"),
                    _band(row, "puzzles", "1500-2000"),
                    _band(row, "puzzles", "2000+"),
                    _elo_cell(row),
                    "n/a" if row.get("delta_cp") is None else f"{row['delta_cp']:.1f}",
                    "n/a"
                    if row.get("first_move_entropy") is None
                    else f"{row['first_move_entropy']:.4f}",
                    str(row.get("date") or ""),
                ]
            )
            + " |"
        )
    lines.append("")
    lines.append(
        f"{len(playing)} etapas medidas con la misma suite. Las filas del encoder viven aparte "
        "porque no comparten una sola columna con estas; las que una medición retiró no están "
        "(`rukh eval drop`)."
    )
    return "\n".join(lines) + "\n"


def write_benchmarks(results: Path | str, out: Path | str) -> int:
    """Rewrite the results section of ``docs/benchmarks.md``; returns how many rows it wrote.

    Everything above ``## Resultados`` is the document's own text -- what each column means and
    how it is measured -- and is left exactly as it is. Only the table is generated.
    """
    payload = json.loads(Path(results).read_text(encoding="utf-8"))
    found = payload.get("rows") if isinstance(payload, dict) else payload
    rows = [item for item in found if isinstance(item, dict)] if isinstance(found, list) else []
    target = Path(out)
    document = target.read_text(encoding="utf-8")
    head, mark, _ = document.partition(BENCHMARKS_MARK)
    if not mark:
        raise ValueError(f"{target} has no `{BENCHMARKS_MARK}` heading to rewrite")
    target.write_text(head + render_benchmarks(rows), encoding="utf-8", newline="\n")
    return len([row for row in rows if row.get("kind") != "encoder"])