Evaluar el encoder: qué se mide y qué sale

"""The encoder's own suite: blunder detection against the baseline, value correlation, result.

``rukh eval`` measures a decoder by the moves it plays; an encoder plays nothing, so it is
measured by what it knows about a position it has never seen. The held-out split is the one
``rukh.data.labels`` draws **by game** (two positions of the same game are not independent), and
every number here is computed on the very same rows for the model and for
``rukh.eval.heuristic``, because ``GOAL.md`` asks for a margin between the two and a margin
measured on two different sets is not a margin.

Four questions, four numbers:

``blunder``
    precision, recall and F1 over the rows that have a blunder label at all, for the encoder
    *and* for the material baseline. The headline is the difference in F1 points.

    A blunder is rare (about 3.7 % of the labelled rows) and that changes what an F1 at a
    fixed threshold means. The head's probabilities are not calibrated (nobody calibrated
    them), so at 0.5 a perfectly informative head can fire on nothing at all and score an F1 of
    zero while ranking the positions almost right. Scoring the *threshold* instead of the
    representation is not a measurement, so the rows are cut in two **by game** (the policy of
    ``rukh.data.labels``, for the same reason): the ``tune`` half chooses the threshold that
    maximises F1, and the ``score`` half is where the reported F1, precision and recall are
    measured. The number at the fixed 0.5 is printed next to it, on the same ``score`` half, so
    nothing is hidden, and ``ROC AUC`` and ``average precision`` are printed too because they
    are the two summaries that do not depend on an operating point at all. The baseline is a
    hard yes/no rule with no threshold to tune, which the report says out loud: the comparison
    gives the model a sweep the baseline cannot have.

``value``
    Spearman **and** Pearson between the predicted value and ``tanh(cp / 400)``, the bounded
    score the head was trained on — never raw ``cp``, where a forced mate is ``±9 99x`` and a
    handful of rows would decide Pearson for the whole set. Spearman is the headline (it asks
    only whether the ranking is right, which is what "the model knows which position is better"
    means) and the ``GOAL.md`` bar of 0.80 is checked against it; Pearson is printed next to it
    because the two disagree exactly when the order is right and the scale is not. Spearman is
    Pearson over average ranks, written out here rather than imported: ``scipy`` is not a
    dependency.

``result``
    plain accuracy over the three classes.

``label curve``
    the 10/25/50/100 % points of the fine-tuning run, when the checkpoint carries them.

The heuristic is the slow part (a ``python-chess`` board per row), so its verdicts go through
``rukh.eval.cache`` under a key of their own: they do not depend on the weights, so a second
checkpoint evaluated on the same rows pays nothing for them.
"""

from __future__ import annotations

import json
import logging
import math
import zlib
from collections.abc import Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

import chess
import numpy as np
import polars as pl
from pydantic import BaseModel, ConfigDict, Field

from rukh import paths
from rukh.config import BaseConfig
from rukh.data.labels import LabelsConfig, build_labels
from rukh.eval import heuristic
from rukh.eval.cache import EvalCache, config_sha, file_sha
from rukh.eval.report import (
    ReportPaths,
    encoder_row_of,
    write_report_files,
)

log = logging.getLogger(__name__)

SUITE = "encoder"
HEURISTIC_SUITE = "heuristic"
HEURISTIC_KEY = "material-mobility-v1"
"""Cache key of the baseline: bump it when the heuristic's verdicts change."""
CURVE_KEY = "label_curve"
"""The payload key ``rukh.train.checkpoint.CURVE_KEY`` writes; a test pins the two together."""
GOAL_MARGIN = 5.0
"""F1 points the encoder has to add to the baseline (``GOAL.md``)."""
TUNE_HALF = "tune"
"""Half of the labelled rows the operating point is chosen on, and never scored on."""
SCORE_HALF = "score"
"""Half of the labelled rows every reported blunder number is measured on."""
GOAL_VALUE_CORRELATION = 0.80
"""The second acceptance criterion of ``GOAL.md``: value against Stockfish, at least 0.8.

Measured as **Spearman against the bounded score**, and both halves of that sentence matter.
The target is ``tanh(score / 400)``, so the head cannot reproduce centipawns and was never
asked to: correlating a bounded output against raw ``cp`` would compare a number in ``(-1, 1)``
with one that a forced mate sends to ``±9 99x``, and a handful of mates would decide Pearson for
the whole set. Rank correlation is also the honest reading of "the model knows which position is
better", which is what the bar is about. Pearson on the same bounded pair is reported next to it.
"""
DEFAULT_CONFIG = "encoder.yaml"

class EncoderEvalConfig(BaseConfig):
    """What the encoder suite measures and where it reads and writes."""

    stage: str | None = None
    """Name of the row in the results table; defaults to the checkpoint's run directory."""
    labels: LabelsConfig = Field(default_factory=LabelsConfig)
    split: str = "val"
    """Which side of the by-game split to measure; never ``train`` for a published number."""
    positions: int = 10_000
    """Cap on the rows evaluated; ``0`` means the whole split."""
    batch_size: int = 256
    threshold: float = 0.5
    """Probability above which the encoder's ``blunder`` logit counts as a blunder."""
    mobility_weight: float = heuristic.MOBILITY_WEIGHT
    blunder_material: float = heuristic.BLUNDER_MATERIAL
    seed: int = 42
    out_dir: str = "artifacts/eval"
    web_results: str | None = "artifacts/web/results.json"
    cache_db: str = "artifacts/eval/cache.sqlite"
    device: str | None = None
    """Where the model runs; ``None`` means ``rukh.train.pick_device()`` (CUDA when present)."""
    track: bool = True

    def heuristic_fields(self) -> dict[str, Any]:
        """The settings a cached *baseline* verdict depends on: never the weights.

        ``labels`` is in here even though the verdict is a function of the position alone,
        because it says *which table the positions came from*: pointing ``positions_eval`` at a
        different parquet has to invalidate the cache, and a cached verdict is keyed by the
        position itself (see ``heuristic_predictions``) precisely so that two tables sharing a
        position share the answer instead of overwriting each other's.
        """
        return {
            "heuristic": HEURISTIC_KEY,
            "mobility_weight": self.mobility_weight,
            "blunder_material": self.blunder_material,
            "labels": self.labels.model_dump(mode="json"),
        }

class ClassificationResult(BaseModel):
    """Precision, recall and F1 of one binary detector over one set of items."""

    model_config = ConfigDict(extra="forbid")

    name: str
    items: int
    positives: int
    """Rows whose label is 1."""
    predicted: int
    true_positives: int
    false_positives: int
    false_negatives: int
    precision: float
    recall: float
    f1: float
    accuracy: float

class RankingResult(BaseModel):
    """What a detector's *score* is worth before anyone picks a threshold for it.

    ROC AUC is the probability that a random blunder is ranked above a random quiet move, and
    average precision is the area under the precision-recall curve, which is the right summary
    when the positive class is rare: a coin flip scores 0.5 on the first and the base rate on
    the second, so the two say different things about the same ranking. The span of the
    probabilities is here as well, because it is what makes a fixed threshold reasonable or
    absurd: a head whose largest output is 0.26 fires on nothing at 0.5.
    """

    model_config = ConfigDict(extra="forbid")

    name: str
    items: int
    positives: int
    base_rate: float
    """Share of the rows that are blunders; also what average precision is compared against."""
    roc_auc: float | None = None
    average_precision: float | None = None
    min_score: float | None = None
    max_score: float | None = None
    mean_score: float | None = None

class CorrelationResult(BaseModel):
    """How well a predicted value tracks Stockfish's score."""

    model_config = ConfigDict(extra="forbid")

    name: str
    items: int
    pearson: float | None = None
    spearman: float | None = None
    target: str = "tanh(cp / value_scale)"
    """What the prediction was correlated against; never raw ``cp``.

    See ``GOAL_VALUE_CORRELATION`` for why."""


class CurvePoint(BaseModel):
    """One point of the label-count curve, as the fine-tuning run recorded it."""

    model_config = ConfigDict(extra="forbid")

    fraction: float
    train_labels: int | None = None
    metrics: dict[str, float] = {}

class EncoderResult(BaseModel):
    """Everything one encoder evaluation produced; serialised verbatim as ``results.json``."""

    model_config = ConfigDict(extra="forbid")

    stage: str
    suite: str = SUITE
    checkpoint: str
    model_sha: str
    params: int
    date: str
    device: str = "cpu"
    run_id: str | None = None
    items: int = 0
    blunder_items: int = 0
    """Rows that carry a blunder label; they are split in two halves by ``game_id``."""
    blunder_positives: int = 0
    """How many of those rows are blunders."""
    blunder_base_rate: float | None = None
    """Share of blunders among the labelled rows: the number every metric here has to be read
    against, and the reason accuracy says nothing (see ``BASE_RATE_CAVEAT``)."""
    tune_items: int = 0
    """Rows of the ``tune`` half: where the threshold is chosen and where nothing is reported."""
    score_items: int = 0
    """Rows of the ``score`` half: where every reported blunder number is measured."""
    tune_games: int = 0
    score_games: int = 0
    """Games behind each half; no ``game_id`` is ever in both."""
    threshold_fixed: float | None = None
    """The configured threshold, reported as a second operating point and never as the headline."""
    threshold_tuned: float | None = None
    """The threshold that maximises F1 on the ``tune`` half; the headline is measured with it."""
    tune_f1: float | None = None
    """F1 of the tuned threshold **on the half it was chosen on**: the optimistic number, kept
    next to the honest one so the distance between the two is visible."""
    threshold_split_degenerate: bool = False
    """True when the by-game split could not give both halves blunders, so the threshold had to
    be chosen and measured on the same rows; the report says so and the number is optimistic."""

    encoder_blunder: ClassificationResult | None = None
    """The headline: the encoder at the tuned threshold, on the ``score`` half."""
    encoder_blunder_fixed: ClassificationResult | None = None
    """The same rows at the fixed threshold, so the operating point cannot hide anything."""
    encoder_blunder_ranking: RankingResult | None = None
    """ROC AUC and average precision on the ``score`` half: no threshold involved."""
    heuristic_blunder: ClassificationResult | None = None
    """The baseline on the very same ``score`` half. It is a yes/no rule: nothing was tuned."""
    f1_margin: float | None = None
    """Encoder F1 minus baseline F1, in points; ``GOAL.md`` asks for at least five."""
    meets_goal: bool | None = None
    """The **blunder** criterion alone; ``GOAL.md`` has two and this is the first."""
    encoder_value: CorrelationResult | None = None
    heuristic_value: CorrelationResult | None = None
    value_correlation_meets_goal: bool | None = None
    """The second criterion: Spearman of the value head against the bounded score >= 0.80."""
    meets_all_goals: bool | None = None
    """Both criteria at once, which is what "P3 is done" means."""
    result_accuracy: float | None = None
    label_curve: list[CurvePoint] = []
    notes: list[str] = []
    config: dict[str, Any] = {}

def config_path() -> Path:
    """Where the encoder suite's YAML lives inside the source tree."""
    return paths.package_root() / "configs" / "eval" / DEFAULT_CONFIG


def load_encoder_suite(config: Path | None = None) -> EncoderEvalConfig:
    """Load the encoder suite config: an explicit path wins over the packaged one."""
    from rukh.config import load_yaml

    return load_yaml(config if config is not None else config_path(), EncoderEvalConfig)

def build_items(cfg: EncoderEvalConfig, source: pl.DataFrame | None = None) -> pl.DataFrame:
    """The held-out rows to evaluate, each with the position that preceded it.

    The blunder label judges ``last_move``, the move that *led to* ``fen``, so the baseline needs
    the position before it: the same ``(game_id, ply - 1)`` self-join ``rukh.data.labels`` uses
    to build the label in the first place. Exactly the rows with a predecessor carry a label.
    """
    frame = build_labels(cfg.labels, source)
    previous = frame.select(
        pl.col("game_id"),
        (pl.col("ply") + 1).alias("ply"),
        pl.col("fen").alias("fen_before"),
    )
    items = (
        frame.join(previous, on=["game_id", "ply"], how="left")
        .filter(pl.col("split") == cfg.split)
        .sort(["order", "game_id", "ply"])
    )
    return items.head(cfg.positions) if cfg.positions else items

def encoder_items(items: pl.DataFrame, model: Any, labels: LabelsConfig) -> list[list[int]]:
    """Tokenize the rows in the scheme the model was trained on, whichever that is.

    ``squares`` reads the FEN and nothing else; ``moves`` needs the line that reached the
    position, so ``rukh.data.labels.game_moves`` joins the P1 games back in exactly as
    ``rukh.train.heads`` does — the evaluation has to feed the model the shape it was fine-tuned
    on, and hard-coding ``fen_to_tokens`` here would silently evaluate a ``moves`` encoder on
    tokens from another vocabulary.
    """
    from rukh.data.labels import game_moves
    from rukh.train.heads import LabelledPositions

    scheme = str(model.encoder.cfg.input)
    moves = game_moves(items, labels.games_dir) if scheme == "moves" else None
    dataset = LabelledPositions(items, scheme, moves, model.encoder.cfg.block)
    return [dataset.tokens(index) for index in range(len(dataset))]

def predict(
    model: Any,
    items: Sequence[Sequence[int]],
    batch_size: int = 256,
    device: str | None = None,
) -> dict[str, np.ndarray]:
    """Run the three heads over the tokenized rows: ``value``, ``blunder`` (a probability) and
    ``result``; short sequences are padded and the padding is masked out."""
    import torch

    from rukh.train.heads import collate

    where = torch.device(device or "cpu")
    values: list[np.ndarray] = []
    blunders: list[np.ndarray] = []
    results: list[np.ndarray] = []
    with torch.no_grad():
        for start in range(0, len(items), max(1, batch_size)):
            chunk = [list(tokens) for tokens in items[start : start + max(1, batch_size)]]
            batch = collate(
                [{"idx": torch.tensor(tokens, dtype=torch.long)} for tokens in chunk]  # type: ignore[misc]
            )
            outputs = model(batch["idx"].to(where), batch["attention_mask"].to(where))
            values.append(outputs["value"].detach().float().cpu().numpy())
            blunders.append(torch.sigmoid(outputs["blunder"].detach().float()).cpu().numpy())
            results.append(outputs["result"].detach().float().argmax(dim=-1).cpu().numpy())
    empty = np.zeros(0, dtype=np.float32)
    return {
        "value": np.concatenate(values) if values else empty,
        "blunder": np.concatenate(blunders) if blunders else empty,
        "result": np.concatenate(results) if results else np.zeros(0, dtype=np.int64),
    }

def _verdict(fen_before: str, move: str, cfg: EncoderEvalConfig) -> tuple[int, float]:
    verdict = heuristic.judge(fen_before, move, cfg.blunder_material, cfg.mobility_weight)
    return int(verdict.blunder), verdict.loss


def heuristic_predictions(
    items: pl.DataFrame, cfg: EncoderEvalConfig, cache: EvalCache | None = None
) -> dict[str, np.ndarray]:
    """The baseline's value for every row and its blunder call wherever a move can be judged.

    A row whose predecessor is missing gets ``nan`` for the blunder call, exactly like its label:
    the two sides of the comparison skip the same rows.
    """
    values = np.full(items.height, np.nan, dtype=np.float64)
    calls = np.full(items.height, np.nan, dtype=np.float64)
    columns = items.select("game_id", "ply", "fen", "fen_before", "last_move").rows()
    for index, (_game_id, _ply, fen, fen_before, last_move) in enumerate(columns):
        # Keyed by the position judged, never by ``game_id:ply``: the payload is a function of
        # these three strings and of nothing else, so the same position reached from another
        # table reuses the answer, and a *different* position that happens to sit at the same
        # ply of the same game cannot silently inherit a stale verdict. ``fen_before`` is part
        # of the key because ``fen`` plus ``last_move`` does not recover the captured piece,
        # and the baseline's whole judgement is the material difference between the two.
        item_id = f"{fen}|{fen_before or ''}|{last_move or ''}"
        payload = cache.get(HEURISTIC_SUITE, item_id) if cache is not None else None
        if payload is None:
            board_value = heuristic.value(chess.Board(fen), cfg.mobility_weight)
            call: float | None = None
            if fen_before and last_move:
                call = float(_verdict(str(fen_before), str(last_move), cfg)[0])
            payload = {"value": board_value, "blunder": call}
            if cache is not None:
                cache.put(HEURISTIC_SUITE, item_id, payload)
        values[index] = float(payload["value"])
        called = payload.get("blunder")
        calls[index] = np.nan if called is None else float(called)
    return {"value": values, "blunder": calls}

def _heuristic_cache(cfg: EncoderEvalConfig, use_cache: bool) -> EvalCache:
    """The one cache this suite has: the baseline's verdicts, which outlive the weights.

    There is deliberately no cache for the model's own predictions. A forward pass over ten
    thousand positions is seconds, the heuristic is a ``python-chess`` board per row, and a
    second cache keyed by the weights would only ever be read when the very same checkpoint is
    evaluated twice on the very same rows.
    """
    database = paths.resolve(cfg.cache_db) if use_cache else None
    return EvalCache(
        database,
        HEURISTIC_KEY,
        enabled=use_cache,
        config_sha=config_sha(cfg.heuristic_fields()),
    )

def label_curve_points(payload: dict[str, Any]) -> list[CurvePoint]:
    """The label-count curve a fine-tuning run may have written into its checkpoint.

    ``rukh train heads --curve`` trains one run per fraction; when the run that produced this
    checkpoint recorded them, they travel in the payload (or in its ``cfg``) under
    ``label_curve`` and are reported as they are. Nothing is invented when they are absent.
    """
    found = payload.get(CURVE_KEY)
    if found is None:
        found = (payload.get("cfg") or {}).get(CURVE_KEY)
    if not isinstance(found, list):
        return []
    points: list[CurvePoint] = []
    for entry in found:
        if not isinstance(entry, dict) or "fraction" not in entry:
            continue
        metrics = entry.get("metrics") or {}
        points.append(
            CurvePoint(
                fraction=float(entry["fraction"]),
                train_labels=(
                    int(entry["train_labels"]) if entry.get("train_labels") is not None else None
                ),
                metrics={
                    str(key): float(value)
                    for key, value in metrics.items()
                    if isinstance(value, (int, float))
                },
            )
        )
    return sorted(points, key=lambda point: point.fraction)

class EncoderWebRow(BaseModel):
    """One row of the same table for a model that judges positions instead of playing them.

    An encoder has no legality, no Elo and no puzzles, and a decoder has no blunder F1: forcing
    the two into one schema would fill the table with columns that are structurally ``null``.
    The rows share the key (``stage``) and nothing else, and this one says so with ``kind``; a
    row without a ``kind`` is a decoder row, which is what every row written before M3 is.
    """

    model_config = ConfigDict(extra="forbid")

    stage: str
    kind: Literal["encoder"] = "encoder"
    params: int
    blunder_f1: float | None = None
    """F1 of the blunder head at its tuned operating point, on the half it was not tuned on."""
    blunder_f1_fixed: float | None = None
    """F1 of the same head on the same rows at the configured threshold, usually 0.5.

    On a class this rare the two differ by a lot, and publishing only one of them would be
    publishing a threshold instead of a model."""
    blunder_threshold: float | None = None
    """The threshold behind ``blunder_f1``, chosen on the other half of the labelled rows."""
    blunder_f1_heuristic: float | None = None
    """F1 of the material baseline on the very same rows. It is a rule: nothing was tuned."""
    blunder_f1_margin: float | None = None
    """The difference above, in F1 points; ``GOAL.md`` asks for five."""
    blunder_precision: float | None = None
    blunder_recall: float | None = None
    blunder_roc_auc: float | None = None
    """Threshold-free: the probability that a blunder is ranked above a quiet move."""
    blunder_average_precision: float | None = None
    """Threshold-free: area under the precision-recall curve; a coin flip scores the base rate."""
    blunder_base_rate: float | None = None
    """Share of blunders among the scored rows; without it the two numbers above mean nothing."""
    value_pearson: float | None = None
    value_spearman: float | None = None
    result_accuracy: float | None = None
    positions: int = 0
    date: str
    run_id: str | None = None


TableRow = WebRow | EncoderWebRow
"""What ``upsert_row`` accepts: one row of the single results table, of either kind."""

def encoder_row_of(result: EncoderResult) -> EncoderWebRow:
    """The table row a finished encoder evaluation produces."""
    encoder = result.encoder_blunder
    fixed = result.encoder_blunder_fixed
    baseline = result.heuristic_blunder
    measured = result.encoder_blunder_ranking
    value = result.encoder_value
    return EncoderWebRow(
        stage=result.stage,
        params=result.params,
        blunder_f1=encoder.f1 if encoder else None,
        blunder_f1_fixed=fixed.f1 if fixed else None,
        blunder_threshold=result.threshold_tuned,
        blunder_f1_heuristic=baseline.f1 if baseline else None,
        blunder_f1_margin=result.f1_margin,
        blunder_precision=encoder.precision if encoder else None,
        blunder_recall=encoder.recall if encoder else None,
        blunder_roc_auc=measured.roc_auc if measured else None,
        blunder_average_precision=measured.average_precision if measured else None,
        blunder_base_rate=result.blunder_base_rate,
        value_pearson=value.pearson if value else None,
        value_spearman=value.spearman if value else None,
        result_accuracy=result.result_accuracy,
        positions=result.items,
        date=result.date,
        run_id=result.run_id,
    )

def write_report_files(
    stage: str,
    markdown_text: str,
    payload: Any,
    out_dir: Path,
    web_results: Path | None,
    row: TableRow,
) -> ReportPaths:
    """Write one stage's ``report.md`` and ``results.json`` and upsert its row for the web.

    The two suites (the decoder's and the encoder's) measure different things and render
    different reports, but they write them in the same place, in the same shape and into the
    same table, so the part that is the same lives here once.
    """
    directory = Path(out_dir) / stage
    directory.mkdir(parents=True, exist_ok=True)
    markdown = directory / REPORT_NAME
    markdown.write_text(markdown_text, encoding="utf-8", newline="\n")
    results = directory / RESULTS_NAME
    results.write_text(
        json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline="\n"
    )
    web: str | None = None
    if web_results is not None:
        upsert_row(Path(web_results), row)
        web = Path(web_results).as_posix()
    return ReportPaths(markdown=markdown.as_posix(), results=results.as_posix(), web=web)

# Encoder suite (docs/spec/02, component 2): blunder detection against the material baseline,
# the correlation of the value head with Stockfish's `cp`, and the accuracy of the result head.
# Everything is measured on the held-out split of `rukh.data.labels`, which is drawn by game.
stage: null # defaults to the checkpoint's run directory
split: val # never `train` for a published number
positions: 10000 # 0 evaluates the whole split
batch_size: 256
# The fixed operating point, reported next to the tuned one and never as the headline: a
# blunder is ~3.7 % of the rows and the head's sigmoid is not calibrated, so the F1 at 0.5
# measures the threshold rather than the model. The reported F1 uses the threshold that
# maximises F1 on the `tune` half of the labelled rows (split by game_id, like everything
# else here) and is measured on the `score` half, together with ROC AUC and average precision.
threshold: 0.5 # probability above which the blunder logit counts as a blunder
# The baseline of GOAL.md: material (1/3/3/5/9) plus mobility, one ply of captures ahead.
mobility_weight: 0.05
blunder_material: 1.0
labels:
  positions_eval: data/evals/positions-eval.parquet
  out_dir: data/labels
  value_scale: 400.0
  blunder_cp: 100
  val_fraction: 0.1
  seed: 42
seed: 42
out_dir: artifacts/eval
web_results: artifacts/web/results.json
cache_db: artifacts/eval/cache.sqlite
# device: null -> CUDA when it is available (rukh.train.pick_device)
track: true

eval_app = typer.Typer(
    help="Evaluate a model: the decoder by default, a subcommand for the encoder.",
    invoke_without_command=True,
)
app.add_typer(eval_app, name="eval")

@eval_app.callback(invoke_without_command=True)
def eval_cmd(
    ctx: typer.Context,
    model: Annotated[
        str | None,
        typer.Option("--model", help="Checkpoint path or Hub id (owner/name) to evaluate."),
    ] = None,
    suite: Annotated[str, typer.Option("--suite", help="Suite name: full or quick.")] = "full",
    config: Annotated[
        Path | None,
        typer.Option(
            "--config", exists=True, dir_okay=False, readable=True, help="Suite YAML override."
        ),
    ] = None,
    stage: Annotated[
        str | None, typer.Option("--stage", help="Row name in the results table.")
    ] = None,
    no_cache: Annotated[
        bool, typer.Option("--no-cache", help="Recompute every game and puzzle.")
    ] = False,
    device: Annotated[
        str | None,
        typer.Option("--device", help="Where to run: cuda, cpu... (default: the training device)."),
    ] = None,
    as_json: Annotated[bool, typer.Option("--json", help="Print the result as JSON only.")] = False,
) -> None:
    """Measure legality, next-move accuracy, puzzles and Elo, and write the report.

    ``rukh eval --model ...`` is the decoder, exactly as it always was; the subcommands
    evaluate the other models (``rukh eval encoder``).
    """
    from rukh.eval import load_suite, run_suite
    from rukh.eval.report import elo_line
    from rukh.eval.suite import SUITES, is_hub_id

    if ctx.invoked_subcommand is not None:
        return
    if model is None:
        typer.echo("error: --model is required (see rukh eval --help)", err=True)
        raise typer.Exit(code=2)

@eval_app.command("encoder")
def eval_encoder_cmd(
    model: Annotated[
        Path,
        typer.Option(
            "--model", exists=True, dir_okay=False, readable=True, help="Fine-tuned checkpoint."
        ),
    ],
    config: Annotated[
        Path | None,
        typer.Option(
            "--config", exists=True, dir_okay=False, readable=True, help="Suite YAML override."
        ),
    ] = None,
    stage: Annotated[
        str | None, typer.Option("--stage", help="Row name in the results table.")
    ] = None,
    device: Annotated[
        str | None,
        typer.Option("--device", help="Where to run: cuda, cpu... (default: the training device)."),
    ] = None,
    no_cache: Annotated[
        bool, typer.Option("--no-cache", help="Recompute every baseline verdict.")
    ] = False,
    as_json: Annotated[bool, typer.Option("--json", help="Print the result as JSON only.")] = False,
) -> None:
    """Measure the encoder's heads against the labels and the material baseline."""
    from rukh.eval.encoder import load_encoder_suite, run_encoder_suite

    logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(message)s")
    cfg = load_encoder_suite(config)
    if stage is not None:
        cfg = cfg.model_copy(update={"stage": stage})
    try:
        result, report = run_encoder_suite(model, cfg, use_cache=not no_cache, device=device)
    except (FileNotFoundError, ValueError) as exc:
        typer.echo(f"error: {exc}", err=True)
        raise typer.Exit(code=1) from exc
    if as_json:
        typer.echo(result.model_dump_json(indent=2))
        return

    typer.echo(f"stage:    {result.stage} ({result.params:,} parameters)")
    typer.echo(
        f"items:    {result.items} positions, {result.blunder_items} with a blunder label "
        f"on {result.device}"
    )
    typer.echo(
        f"split:    tune {result.tune_items} rows / {result.tune_games} games, "
        f"score {result.score_items} rows / {result.score_games} games "
        f"(base rate {(result.blunder_base_rate or 0.0) * 100:.2f} %)"
    )
    tuned = "n/a" if result.threshold_tuned is None else f"{result.threshold_tuned:.4g}"
    fixed = "n/a" if result.threshold_fixed is None else f"{result.threshold_fixed:.4g}"
    for measured, point in (
        (result.encoder_blunder, f"p>={tuned} tuned"),
        (result.encoder_blunder_fixed, f"p>={fixed} fixed"),
        (result.heuristic_blunder, "rule, untuned"),
    ):
        if measured is not None:
            typer.echo(
                f"blunder:  {measured.name:<15} {point:<16} P {measured.precision:.4f}  "
                f"R {measured.recall:.4f}  F1 {measured.f1:.4f}"
            )
    if result.encoder_blunder_ranking is not None:
        rank = result.encoder_blunder_ranking
        auc = "n/a" if rank.roc_auc is None else f"{rank.roc_auc:.4f}"
        average = "n/a" if rank.average_precision is None else f"{rank.average_precision:.4f}"
        typer.echo(
            f"ranking:  roc auc {auc}  average precision {average}  "
            f"(a coin flip scores 0.5000 and {rank.base_rate:.4f})"
        )

    if result.f1_margin is not None:
        verdict = "meets the bar" if result.meets_goal else "below the bar"
        typer.echo(f"margin:   {result.f1_margin:+.1f} F1 points over the baseline ({verdict})")
    if result.encoder_value is not None:
        from rukh.eval.encoder import GOAL_VALUE_CORRELATION

        met = result.value_correlation_meets_goal
        verdict = "not measured" if met is None else ("meets the bar" if met else "below the bar")
        typer.echo(
            f"value:    vs {result.encoder_value.target}  "
            f"spearman {result.encoder_value.spearman}  "
            f"pearson {result.encoder_value.pearson}  "
            f"(>= {GOAL_VALUE_CORRELATION:.2f}: {verdict})"
        )
    if result.result_accuracy is not None:
        typer.echo(f"result:   {result.result_accuracy:.4f} accuracy")
    for point in result.label_curve:
        typer.echo(f"curve:    {point.fraction:>5.0%} of the labels ({point.train_labels} rows)")
    for note in result.notes:
        typer.echo(f"note:     {note}")
    typer.echo(f"report:   {report.markdown}")
    typer.echo(f"results:  {report.results}")
    if report.web:
        typer.echo(f"table:    {report.web}")

def write_encoder_report(
    result: EncoderResult, out_dir: Path, web_results: Path | None = None
) -> ReportPaths:
    """Write ``report.md`` and ``results.json`` and upsert the encoder's row for the web."""
    return write_report_files(
        result.stage,
        render_markdown(result),
        json.loads(result.model_dump_json()),
        Path(out_dir),
        web_results,
        encoder_row_of(result),
    )


def run_encoder_suite(
    ckpt: Path | str,
    cfg: EncoderEvalConfig,
    use_cache: bool = True,
    device: str | None = None,
) -> tuple[EncoderResult, ReportPaths]:
    """Evaluate, track and report: the whole of ``rukh eval encoder`` in one call."""
    result = evaluate_encoder(ckpt, cfg, use_cache=use_cache, device=device)
    if cfg.track:
        result.run_id = track_result(result, cfg)
    report = write_encoder_report(
        result,
        paths.resolve(cfg.out_dir),
        paths.resolve(cfg.web_results) if cfg.web_results else None,
    )
    return result, report

from rukh.eval.encoder import (
    ClassificationResult,
    CorrelationResult,
    CurvePoint,
    EncoderEvalConfig,
    EncoderResult,
    classification,
    evaluate_encoder,
    load_encoder_suite,
    pearson,
    ranks,
    run_encoder_suite,
    spearman,
)

from rukh.eval.report import (
    EncoderWebRow,
    ReportPaths,
    WebRow,
    elo_line,
    encoder_row_of,
    render_markdown,
    row_of,
    upsert_row,
    write_report,
)

    "file_sha",
    "fit_elo",
    "hub_checkpoint",
    "is_hub_id",
    "legality",
    "load_encoder_suite",
    "load_puzzles",
    "load_suite",
    "model_source",
    "one_sided_bound",
    "pearson",
    "play_rung",
    "play_rungs",
    "position_at",
    "ranks",
    "record_of",
    "render_markdown",
    "resolve_model",
    "row_of",
    "run_encoder_suite",
    "run_puzzles",
    "run_suite",
    "sample_positions",
    "score_of",
    "separation",
    "solve_puzzle",
    "spearman",
    "upsert_row",
    "write_report",
]