El encoder: medir sin engañarse

def base_rate_caveat(base_rate: float | None) -> str:
    """The one sentence that has to travel with every number in this section.

    It is in the report and in the model card because it is the transferable lesson of the
    module: on a rare class, the two numbers everybody reaches for first say nothing.
    """
    share = "the blunder base rate" if base_rate is None else f"{base_rate * 100:.1f} %"
    always = "" if base_rate is None else f" scores {(1 - base_rate) * 100:.1f} %"
    return (
        f"a blunder is rare ({share} of the labelled rows), so **accuracy is meaningless** here: "
        f'a model that always answers "no blunder"{always} without knowing anything about '
        "chess. F1 at an arbitrary threshold is nearly as bad, because an uncalibrated sigmoid "
        "can rank the positions well and still put every probability below 0.5: that number "
        "measures the operating point, not the representation. This is why the threshold is "
        f"chosen on a `{TUNE_HALF}` half and the F1 is reported on a `{SCORE_HALF}` half, and "
        "why ROC AUC and average precision, which no threshold can flatter, are reported next "
        "to it"
    )


def sentence(text: str) -> str:
    """The same clause as a sentence: the notes start in lower case, the prose does not.

    ``str.capitalize`` is not it: it would lower-case ``ROC AUC`` and ``F1`` on the way past.
    """
    text = text.strip()
    return text[:1].upper() + text[1:] + ("" if text.endswith(".") else ".")

def threshold_half(game_id: int, seed: int) -> str:
    """Which half of the labelled rows a game falls on: ``tune`` or ``score``.

    By game and never by position, exactly like ``rukh.data.labels.game_split`` and for exactly
    the same reason: two positions of the same game are one move apart, so choosing a threshold
    on one and scoring it on the other would tune on the rows being scored through the back
    door. CRC-32 of a salted string, so the halves are the same in every process and every run,
    and the salt keeps this split independent of the train/val one.
    """
    return TUNE_HALF if zlib.crc32(f"{seed}:threshold:{game_id}".encode()) % 2 == 0 else SCORE_HALF


def threshold_halves(games: Sequence[int], seed: int) -> np.ndarray:
    """Boolean mask of the rows that belong to the ``tune`` half, one entry per row."""
    return np.array([threshold_half(int(game), seed) == TUNE_HALF for game in games], dtype=bool)

def f1_at(truth: Sequence[int], scores: Sequence[float], threshold: float) -> float:
    """F1 of ``scores >= threshold`` against ``truth``."""
    calls = (np.asarray(scores, dtype=np.float64) >= threshold).astype(np.int64)
    return classification(truth, calls).f1


def best_threshold(
    truth: Sequence[int], scores: Sequence[float], include: float = 0.5
) -> tuple[float, float]:
    """The threshold that maximises F1 on these rows, and the F1 it reaches.

    The candidates are the distinct scores (every threshold that can change a single call) plus
    ``include``, the configured one, so the answer is never *worse* than the fixed threshold on
    the rows it was chosen on — which is the only guarantee a sweep can honestly give, and the
    reason the reported number is measured somewhere else. Ties go to the lowest threshold: on a
    rare class, the lower one flags more and is the less lucky of the two.
    """
    labels = np.asarray(truth, dtype=np.int64)
    values = np.asarray(scores, dtype=np.float64)
    if len(labels) != len(values):
        raise ValueError(f"{len(labels)} labels against {len(values)} scores")
    positives = int(np.sum(labels == 1))
    if not len(labels) or not positives:
        return float(include), f1_at(labels, values, include)
    order = np.argsort(-values, kind="stable")
    ordered = values[order]
    hits = np.cumsum(labels[order] == 1)
    seen = np.arange(1, len(labels) + 1, dtype=np.float64)
    last = np.append(ordered[1:] != ordered[:-1], True)
    recall = hits[last] / positives
    precision = hits[last] / seen[last]
    total = precision + recall
    f1 = np.where(total > 0, 2 * precision * recall / np.where(total > 0, total, 1.0), 0.0)
    candidates = [
        (float(value), float(score)) for value, score in zip(ordered[last], f1, strict=True)
    ]
    candidates.append((float(include), f1_at(labels, values, include)))
    candidates.sort(key=lambda pair: (-pair[1], pair[0]))
    return candidates[0]

def classification(
    truth: Sequence[int], predicted: Sequence[int], name: str = "blunder"
) -> ClassificationResult:
    """Precision, recall and F1 of ``predicted`` against ``truth`` (both 0/1 per row)."""
    if len(truth) != len(predicted):
        raise ValueError(f"{len(truth)} labels against {len(predicted)} predictions")
    labels = np.asarray(truth, dtype=np.int64)
    calls = np.asarray(predicted, dtype=np.int64)
    hits = int(np.sum((labels == 1) & (calls == 1)))
    false_positives = int(np.sum((labels == 0) & (calls == 1)))
    false_negatives = int(np.sum((labels == 1) & (calls == 0)))
    precision = hits / (hits + false_positives) if hits + false_positives else 0.0
    recall = hits / (hits + false_negatives) if hits + false_negatives else 0.0
    f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
    correct = int(np.sum(labels == calls))
    return ClassificationResult(
        name=name,
        items=len(labels),
        positives=int(np.sum(labels == 1)),
        predicted=int(np.sum(calls == 1)),
        true_positives=hits,
        false_positives=false_positives,
        false_negatives=false_negatives,
        precision=precision,
        recall=recall,
        f1=f1,
        accuracy=correct / len(labels) if len(labels) else 0.0,
    )

def roc_auc(truth: Sequence[int], scores: Sequence[float]) -> float | None:
    """Area under the ROC curve, as the rank sum of the positives (Mann-Whitney U).

    Written out rather than imported: ``scipy`` and ``scikit-learn`` are not dependencies. Ties
    are handled by the average ranks of ``ranks``, which is what gives a constant predictor
    exactly 0.5 instead of 0 or 1 depending on how the sort happened to break the tie.
    """
    labels = np.asarray(truth, dtype=np.int64)
    values = np.asarray(scores, dtype=np.float64)
    if len(labels) != len(values):
        raise ValueError(f"{len(labels)} labels against {len(values)} scores")
    positives = int(np.sum(labels == 1))
    negatives = int(np.sum(labels == 0))
    if not positives or not negatives:
        return None
    rank = np.asarray(ranks(values.tolist()), dtype=np.float64)
    rank_sum = float(rank[labels == 1].sum())
    return (rank_sum - positives * (positives + 1) / 2.0) / (positives * negatives)

def average_precision(truth: Sequence[int], scores: Sequence[float]) -> float | None:
    """Area under the precision-recall curve, summed over the steps of the recall.

    ``sum (R_n - R_{n-1}) * P_n`` over the distinct scores, taken from the highest down. Rows
    that share a score are one step, because a threshold cannot separate them; without that,
    a predictor that outputs the same number everywhere would score 1.0 by sorting luck. A
    random ranking scores the base rate, which is why this is the summary that means something
    when the positive class is rare.
    """
    labels = np.asarray(truth, dtype=np.int64)
    values = np.asarray(scores, dtype=np.float64)
    if len(labels) != len(values):
        raise ValueError(f"{len(labels)} labels against {len(values)} scores")
    positives = int(np.sum(labels == 1))
    if not positives:
        return None
    order = np.argsort(-values, kind="stable")
    ordered = values[order]
    hits = np.cumsum(labels[order] == 1)
    seen = np.arange(1, len(labels) + 1, dtype=np.float64)
    last = np.append(ordered[1:] != ordered[:-1], True)  # the end of each group of equal scores
    recall = hits[last] / positives
    precision = hits[last] / seen[last]
    steps = np.diff(np.concatenate(([0.0], recall)))
    return float(np.sum(steps * precision))

def ranking(truth: Sequence[int], scores: Sequence[float], name: str = "encoder") -> RankingResult:
    """Everything about a detector's ranking that no threshold can change."""
    labels = np.asarray(truth, dtype=np.int64)
    values = np.asarray(scores, dtype=np.float64)
    positives = int(np.sum(labels == 1))
    return RankingResult(
        name=name,
        items=len(labels),
        positives=positives,
        base_rate=positives / len(labels) if len(labels) else 0.0,
        roc_auc=roc_auc(labels, values),
        average_precision=average_precision(labels, values),
        min_score=float(values.min()) if len(values) else None,
        max_score=float(values.max()) if len(values) else None,
        mean_score=float(values.mean()) if len(values) else None,
    )

def _finite(*arrays: np.ndarray) -> np.ndarray:
    """Mask of the rows where every array is a real number."""
    mask = np.ones(len(arrays[0]), dtype=bool)
    for array in arrays:
        mask &= np.isfinite(array)
    return mask


def measure_blunder(
    truth: np.ndarray,
    probability: np.ndarray,
    baseline_calls: np.ndarray,
    games: Sequence[int],
    cfg: EncoderEvalConfig,
    result: EncoderResult,
) -> EncoderResult:
    """Score the blunder detector at an operating point that was not chosen on these rows.

    The labelled rows are cut in two **by game**: ``tune`` chooses the threshold that maximises
    F1, ``score`` is where the F1, the precision and the recall that go into the report and into
    the ``GOAL.md`` comparison are measured. The baseline is measured on the same ``score`` rows;
    it has no threshold to tune, so the comparison hands the model a sweep the rule cannot have,
    and the report says so rather than leaving the reader to notice.

    When the split cannot give both halves a blunder — a handful of games, or every blunder in
    one of them — there is no honest way to hold rows back: the threshold is then chosen and
    measured on the same rows, ``threshold_split_degenerate`` is set and the report prints the
    warning next to the number.
    """
    result.blunder_items = len(truth)
    result.blunder_positives = int(np.sum(truth == 1))
    result.blunder_base_rate = float(np.mean(truth == 1)) if len(truth) else None
    result.threshold_fixed = float(cfg.threshold)
    tune = threshold_halves(games, cfg.seed)
    score = ~tune
    identifiers = np.asarray(games)
    usable = bool(
        truth[tune].sum()
        and truth[score].sum()
        and (truth[tune] == 0).any()
        and (truth[score] == 0).any()
    )
    if not usable:
        result.threshold_split_degenerate = True
        tune = np.ones(len(truth), dtype=bool)
        score = np.ones(len(truth), dtype=bool)
    result.tune_items = int(tune.sum())
    result.score_items = int(score.sum())
    result.tune_games = int(len(np.unique(identifiers[tune])))
    result.score_games = int(len(np.unique(identifiers[score])))
    threshold, tune_f1 = best_threshold(truth[tune], probability[tune], cfg.threshold)
    result.threshold_tuned = threshold
    result.tune_f1 = tune_f1
    result.encoder_blunder = classification(
        truth[score], (probability[score] >= threshold).astype(np.int64), "encoder"
    )
    result.encoder_blunder_fixed = classification(
        truth[score], (probability[score] >= cfg.threshold).astype(np.int64), "encoder"
    )
    result.encoder_blunder_ranking = ranking(truth[score], probability[score], "encoder")
    result.heuristic_blunder = classification(truth[score], baseline_calls[score], "heuristic")
    margin = 100.0 * (result.encoder_blunder.f1 - result.heuristic_blunder.f1)
    result.f1_margin = margin
    result.meets_goal = margin >= GOAL_MARGIN
    return result

"""The dumb baseline the encoder has to beat: material, mobility and a one-ply look at captures.

``GOAL.md`` asks for a blunder F1 five points above a heuristic, and the heuristic is only worth
anything as a comparison if it is measured exactly like the model: same positions, same labels,
same definition of a hit. So it lives here, next to the suite, and is deliberately kept as
simple as a chess program can be:

* **material**: pawn 1, knight 3, bishop 3, rook 5, queen 9, king 0 — the values every beginner
  is taught, unchanged by the phase of the game;
* **mobility**: how many legal moves each side has, worth ``MOBILITY_WEIGHT`` of a pawn each,
  which is what turns a pile of material into something that moves with the position;
* **blunder**: the move that led here lost at least ``BLUNDER_MATERIAL`` point of net material,
  measured from the mover's point of view, by playing the move on a ``python-chess`` board and
  letting the opponent answer with its single most profitable capture or promotion.

What it cannot see
------------------

Everything that is not material. The named example the tests use is **Byrne–Fischer, New York
1956** (the "Game of the Century"), after 17.Kf1: Fischer played 17...Be6, offering the queen,
and the game is a win for Black. The heuristic sees 18.Bxb6 taking a queen for nothing and calls
one of the most famous moves in chess a blunder. A positional sacrifice is invisible to it, and
that is the point: a baseline that already understood sacrifices would not be a baseline.

It is also blind in the other direction. The opponent's reply is searched one ply deep and only
over captures and promotions, so our own recapture is never counted: an equal trade whose
recapture comes two plies later reads as a loss. And a piece that was already hanging before the
move is charged again to every quiet move that follows it, because "before" is the material on
the board and not the best the mover could have kept. Both are written down here rather than
fixed, because fixing them would make the baseline a small engine instead of a floor.
"""

from __future__ import annotations

import math
from collections.abc import Iterator

import chess
from pydantic import BaseModel, ConfigDict

PIECE_VALUES: dict[chess.PieceType, float] = {
    chess.PAWN: 1.0,
    chess.KNIGHT: 3.0,
    chess.BISHOP: 3.0,
    chess.ROOK: 5.0,
    chess.QUEEN: 9.0,
    chess.KING: 0.0,
}
"""The beginner's table, in pawns. The king is worth nothing: it is never captured."""

MOBILITY_WEIGHT = 0.05
"""Pawns per legal move of difference; twenty extra moves are worth one pawn."""

BLUNDER_MATERIAL = 1.0
"""Net material a move has to lose, in pawns, before the baseline calls it a blunder."""

VALUE_SCALE = 4.0
"""``tanh(score / 4)`` in pawns is exactly ``tanh(cp / 400)``, the label's own scale."""

GAME_OF_THE_CENTURY = "r3r1k1/pp3pbp/1qp3p1/2B5/2BP2b1/Q1n2N2/P4PPP/3R1K1R b - - 3 17"
"""Byrne–Fischer, New York 1956, after 17.Kf1: the position of the positional sacrifice."""

POSITIONAL_SACRIFICE = "g4e6"
"""17...Be6, the queen offer the heuristic reads as a nine-point blunder."""

class Verdict(BaseModel):
    """What the baseline says about one move: a value, a material loss and a verdict."""

    model_config = ConfigDict(extra="forbid")

    move: str
    blunder: bool
    loss: float
    """Net material the mover lost, in pawns; negative when the move won material."""
    material_before: float
    material_after: float
    """Material after the move and the opponent's most profitable capture."""
    value: float
    """Value of the resulting position, ``tanh(score / 4)`` from White's point of view."""

def material(board: chess.Board) -> float:
    """Material on the board in pawns, from White's point of view."""
    total = 0.0
    for piece_type, value in PIECE_VALUES.items():
        if not value:
            continue
        total += value * len(board.pieces(piece_type, chess.WHITE))
        total -= value * len(board.pieces(piece_type, chess.BLACK))
    return total


def material_for(board: chess.Board, color: chess.Color) -> float:
    """Material from ``color``'s point of view: its own minus the opponent's."""
    return material(board) if color == chess.WHITE else -material(board)


def _moves_for(board: chess.Board, color: chess.Color) -> int:
    """How many legal moves ``color`` has, whoever is to move."""
    if board.turn == color:
        return board.legal_moves.count()
    swapped = board.copy(stack=False)
    swapped.turn = color
    swapped.ep_square = None
    return swapped.legal_moves.count()


def mobility(board: chess.Board) -> int:
    """White's legal moves minus Black's, counted on the same position."""
    return _moves_for(board, chess.WHITE) - _moves_for(board, chess.BLACK)


def score(board: chess.Board, mobility_weight: float = MOBILITY_WEIGHT) -> float:
    """Material plus weighted mobility, in pawns, from White's point of view."""
    return material(board) + mobility_weight * mobility(board)

def value(board: chess.Board, mobility_weight: float = MOBILITY_WEIGHT) -> float:
    """The baseline's value of a position, on the label's own ``tanh`` scale."""
    return math.tanh(score(board, mobility_weight) / VALUE_SCALE)


def value_of(fen: str, mobility_weight: float = MOBILITY_WEIGHT) -> float:
    """``value`` of a FEN (four fields or six)."""
    return value(chess.Board(fen), mobility_weight)

def _material_moves(board: chess.Board) -> Iterator[chess.Move]:
    """The only replies that can change the material count: captures and promotions."""
    yield from board.generate_legal_captures()
    for move in board.legal_moves:
        if move.promotion is not None and not board.is_capture(move):
            yield move


def worst_material(board: chess.Board, color: chess.Color) -> float:
    """``color``'s material after the single most profitable reply the opponent has.

    One ply, captures and promotions only: no recapture, no threat, no check. That is the whole
    search, and the docstring of the module says what it costs.
    """
    worst = material_for(board, color)
    for reply in _material_moves(board):
        child = board.copy(stack=False)
        child.push(reply)
        worst = min(worst, material_for(child, color))
    return worst

def judge(
    fen: str,
    move: str,
    threshold: float = BLUNDER_MATERIAL,
    mobility_weight: float = MOBILITY_WEIGHT,
) -> Verdict:
    """Judge ``move`` played in ``fen``: how much material it lost and whether that is a blunder.

    ``fen`` is the position **before** the move, which is the only way material can be compared;
    the labels of ``rukh.data.labels`` judge the same move from the same side.
    """
    board = chess.Board(fen)
    mover = board.turn
    before = material_for(board, mover)
    board.push(chess.Move.from_uci(move))
    after = worst_material(board, mover)
    loss = before - after
    return Verdict(
        move=move,
        blunder=loss >= threshold,
        loss=loss,
        material_before=before,
        material_after=after,
        value=value(board, mobility_weight),
    )

def test_the_baseline_cannot_see_a_positional_sacrifice() -> None:
    """Byrne-Fischer 1956, 17...Be6: the Game of the Century, scored as a nine-point blunder.

    This is the documented blind spot of the baseline and the reason the encoder is expected to
    beat it: material plus one ply of captures cannot tell a queen sacrifice from a queen loss.
    """
    verdict = judge(GAME_OF_THE_CENTURY, POSITIONAL_SACRIFICE)
    assert verdict.blunder is True
    assert verdict.loss == pytest.approx(9.0)
    # Black is a pawn up before the move and eight points down after 18.Bxb6, so the baseline
    # cannot help calling it: the compensation is not on the board yet. Fischer won the game.
    assert verdict.material_before == pytest.approx(1.0)
    assert verdict.material_after == pytest.approx(-8.0)

HANGING_QUEEN = "r1bqkbnr/pppp1ppp/2n5/4p2Q/4P3/8/PPPP1PPP/RNB1KBNR w KQkq - 2 3"
"""After 1.e4 e5 2.Qh5 Nc6: 3.Qxf7+ wins a pawn and loses the queen to 3...Kxf7."""


def test_the_starting_position_is_perfectly_balanced() -> None:
    board = chess.Board()
    assert material(board) == 0.0
    assert mobility(board) == 0  # twenty moves each
    assert value(board) == 0.0


def test_material_is_counted_from_the_asking_side() -> None:
    board = chess.Board("4k3/8/8/8/8/8/8/3QK3 w - - 0 1")
    assert material(board) == PIECE_VALUES[chess.QUEEN]
    assert material_for(board, chess.WHITE) == 9.0
    assert material_for(board, chess.BLACK) == -9.0


def test_mobility_moves_the_score_without_moving_the_material() -> None:
    # A rook on an open file has more moves than one boxed in, with the same pieces on the board.
    open_file = chess.Board("4k3/8/8/8/8/8/8/R3K3 w - - 0 1")
    boxed_in = chess.Board("4k3/8/8/8/8/8/R7/4K3 w - - 0 1")
    assert material(open_file) == material(boxed_in)
    assert score(open_file) != score(boxed_in)

def test_the_baseline_flags_a_hanging_queen() -> None:
    verdict = judge(HANGING_QUEEN, "h5f7")
    assert verdict.blunder is True
    assert verdict.loss == pytest.approx(8.0)  # a queen for a pawn
    assert verdict.material_before == pytest.approx(0.0)


def test_a_quiet_developing_move_is_not_a_blunder() -> None:
    assert judge(HANGING_QUEEN, "g1f3").blunder is False
    assert judge(HANGING_QUEEN, "g1f3").loss == pytest.approx(0.0)


def test_an_even_trade_is_not_a_blunder() -> None:
    # Ruy Lopez, 4.Bxc6: a bishop for a knight, recaptured at once. The one ply of replies is
    # exactly what keeps an even trade off the blunder list.
    board = chess.Board()
    for san in ("e4", "e5", "Nf3", "Nc6", "Bb5", "Nf6"):
        board.push_san(san)
    verdict = judge(board.fen(), "b5c6")
    assert verdict.blunder is False
    assert verdict.loss == pytest.approx(0.0)


def test_losing_a_piece_for_a_pawn_is_a_blunder() -> None:
    # 1.e4 e5 2.Nf3 Nc6 3.Nxe5?? Nxe5: the knight wins a pawn and is lost for nothing.
    board = chess.Board()
    for san in ("e4", "e5", "Nf3", "Nc6"):
        board.push_san(san)
    verdict = judge(board.fen(), "f3e5")
    assert verdict.blunder is True
    assert verdict.loss == pytest.approx(2.0)  # a knight for a pawn

def test_the_value_is_on_the_label_s_own_scale() -> None:
    # The labels are tanh(cp / 400) and the baseline is tanh(pawns / 4), so four pawns of
    # advantage (400 centipawns) read as tanh(1) on both sides of the comparison.
    four_pawns_up = chess.Board("4k3/8/8/8/8/8/P7/1N2K3 w - - 0 1")
    assert value(four_pawns_up, mobility_weight=0.0) == pytest.approx(math.tanh(1.0), abs=1e-12)
    assert value_of(four_pawns_up.fen(), 0.0) == value(four_pawns_up, 0.0)


def test_a_promotion_is_material_even_when_nothing_is_captured() -> None:
    # Black to move cannot stop the pawn: the mover's quiet king move hands White a queen.
    board = chess.Board("8/P7/8/8/8/8/8/k5K1 b - - 0 1")
    verdict = judge(board.fen(), "a1b1")
    assert verdict.blunder is True
    assert verdict.loss == pytest.approx(8.0)  # a queen appears, minus the pawn that became it

def pearson(x: Sequence[float], y: Sequence[float]) -> float | None:
    """Pearson's correlation, or ``None`` when either side is constant (it is undefined)."""
    if len(x) != len(y):
        raise ValueError(f"{len(x)} values against {len(y)}")
    if len(x) < 2:
        return None
    first = np.asarray(x, dtype=np.float64)
    second = np.asarray(y, dtype=np.float64)
    first = first - first.mean()
    second = second - second.mean()
    denominator = math.sqrt(float(first @ first) * float(second @ second))
    if denominator == 0.0:
        return None
    return float(first @ second / denominator)

def ranks(values: Sequence[float]) -> list[float]:
    """Average ranks, one per value: tied values share the mean of the ranks they occupy."""
    order = sorted(range(len(values)), key=lambda index: values[index])
    out = [0.0] * len(values)
    start = 0
    while start < len(order):
        stop = start
        while stop + 1 < len(order) and values[order[stop + 1]] == values[order[start]]:
            stop += 1
        shared = (start + stop) / 2.0 + 1.0
        for position in range(start, stop + 1):
            out[order[position]] = shared
        start = stop + 1
    return out


def spearman(x: Sequence[float], y: Sequence[float]) -> float | None:
    """Spearman's rank correlation: Pearson over average ranks, written out by hand."""
    if len(x) != len(y):
        raise ValueError(f"{len(x)} values against {len(y)}")
    if len(x) < 2:
        return None
    return pearson(ranks(list(x)), ranks(list(y)))

def correlation(
    predicted: Sequence[float],
    target: Sequence[float],
    name: str,
    target_name: str = "tanh(cp / value_scale)",
) -> CorrelationResult:
    """Both correlations of one predictor against the bounded score, in one record."""
    return CorrelationResult(
        name=name,
        items=len(predicted),
        pearson=pearson(predicted, target),
        spearman=spearman(predicted, target),
        target=target_name,
    )

def measure(
    items: pl.DataFrame,
    predictions: dict[str, np.ndarray],
    baseline: dict[str, np.ndarray],
    cfg: EncoderEvalConfig,
    result: EncoderResult,
) -> EncoderResult:
    """Fill ``result`` with every metric the two predictors produced on the same rows."""
    # The value head is trained on ``tanh(cp / value_scale)`` and is measured against it. Raw
    # ``cp`` would put a mate at +-9 99x against an output that cannot leave (-1, 1), and those
    # few rows would set the Pearson of the whole set on their own.
    bounded = np.tanh(items["cp"].to_numpy().astype(np.float64) / float(cfg.labels.value_scale))
    labels = items["blunder"].cast(pl.Float64).fill_null(np.nan).to_numpy().astype(np.float64)
    result.items = items.height
    scored = _finite(labels, baseline["blunder"])
    result.blunder_items = int(scored.sum())
    if result.blunder_items:
        measure_blunder(
            labels[scored].astype(np.int64),
            predictions["blunder"][scored].astype(np.float64),
            baseline["blunder"][scored].astype(np.int64),
            items["game_id"].to_numpy()[scored],
            cfg,
            result,
        )
    if items.height:
        target = f"tanh(cp / {cfg.labels.value_scale:g})"
        result.encoder_value = correlation(predictions["value"], bounded, "encoder", target)
        result.heuristic_value = correlation(baseline["value"], bounded, "heuristic", target)
        headline = result.encoder_value.spearman
        result.value_correlation_meets_goal = (
            None if headline is None else headline >= GOAL_VALUE_CORRELATION
        )
        truth_result = items["result_class"].to_numpy().astype(np.int64)
        result.result_accuracy = float(np.mean(predictions["result"] == truth_result))
    if result.meets_goal is not None and result.value_correlation_meets_goal is not None:
        result.meets_all_goals = result.meets_goal and result.value_correlation_meets_goal
    return result

def pairwise_rank_loss(pred: Tensor, target: Tensor, margin: float = 0.0) -> Tensor:
    """A differentiable stand-in for "did the ordering come out right?".

    Every pair in the batch votes: when ``target[i] > target[j]`` the prediction should put
    ``i`` above ``j`` too. The vote is a logistic on the predicted difference, weighted by how
    far apart the targets are, so telling a winning position from a losing one matters more than
    splitting two equal ones -- which is also where a static evaluator has no chance without
    search, and where an unweighted ranking loss would spend most of its gradient.

    Returns zero for a batch whose targets are all equal, which has no ordering to learn.
    """
    diff_target = target.unsqueeze(1) - target.unsqueeze(0)
    diff_pred = pred.unsqueeze(1) - pred.unsqueeze(0)
    weight = diff_target.abs()
    sign = torch.sign(diff_target)
    total = weight.sum()
    if not bool(total > 0):
        return torch.zeros((), device=pred.device, dtype=pred.dtype)
    penalty = F.softplus(-(sign * diff_pred - margin))
    return (weight * penalty).sum() / total

def evaluate_encoder(
    ckpt: Path | str,
    cfg: EncoderEvalConfig,
    use_cache: bool = True,
    device: str | None = None,
    source: pl.DataFrame | None = None,
) -> EncoderResult:
    """Measure one fine-tuned encoder; a missing label table becomes a note, never a crash."""
    from rukh.train import load_heads, pick_device

    path = Path(ckpt)
    where = device or cfg.device or pick_device()
    model, payload = load_heads(path, map_location=where)
    model = model.to(where).eval()
    log.info("evaluating %s on %s", path, where)
    notes: list[str] = [
        "the blunder F1 of the encoder and of the material baseline are measured on the same "
        f"rows of the held-out '{cfg.split}' split, which is drawn by game_id, never by "
        f"position, and those rows are cut in two by game_id again: the '{TUNE_HALF}' half "
        f"chooses the threshold and the '{SCORE_HALF}' half is what gets reported"
    ]
    result = EncoderResult(
        stage=cfg.stage or path.parent.name,
        checkpoint=path.as_posix(),
        model_sha=file_sha(path),
        params=sum(parameter.numel() for parameter in model.parameters()),
        date=datetime.now(UTC).date().isoformat(),
        device=str(where),
        label_curve=label_curve_points(payload),
        config=cfg.model_dump(mode="json"),
    )
    if not result.label_curve:
        notes.append(
            "the checkpoint carries no label-count curve: run `rukh train heads --curve` and "
            "evaluate one of its checkpoints to fill that table"
        )

    try:
        items = build_items(cfg, source)
    except FileNotFoundError as exc:
        notes.append(f"no labelled positions ({exc}): every metric was skipped")
        result.notes = notes
        return result
    if not items.height:
        notes.append(f"the '{cfg.split}' split is empty: every metric was skipped")
        result.notes = notes
        return result

    cache = _heuristic_cache(cfg, use_cache)
    try:
        tokenized = encoder_items(items, model, cfg.labels)
        predictions = predict(model, tokenized, cfg.batch_size, str(where))
        baseline = heuristic_predictions(items, cfg, cache)
    finally:
        cache.close()
    measure(items, predictions, baseline, cfg, result)

    if result.f1_margin is not None:
        notes.append(
            f"the encoder is {result.f1_margin:+.1f} F1 points from the baseline; GOAL.md asks "
            f"for at least {GOAL_MARGIN:+.0f}"
        )
    if result.encoder_value is not None:
        notes.append(
            f"the value head is correlated against `{result.encoder_value.target}`, the bounded "
            "score it is trained on, and not against raw `cp`, where a forced mate is worth "
            "±9 99x and a few rows would decide Pearson for the whole set; the GOAL.md bar of "
            f"{GOAL_VALUE_CORRELATION:.2f} is read on Spearman"
        )

    notes.append(
        "the baseline counts material (1/3/3/5/9) and mobility and looks one ply ahead at "
        "captures: it cannot see a positional sacrifice, and calls Fischer's 17...Be6 "
        "(Byrne-Fischer, 1956) a nine-point blunder"
    )
    notes.append(
        "the two blunder detectors do not see the same thing: the baseline is given the "
        "predecessor position and the move that was played, while the encoder is given only the "
        "resulting position and has to infer that something was thrown away. That is the "
        "comparison GOAL.md asks for, but it is not a level playing field"
    )
    notes.extend(blunder_notes(result))
    result.notes = notes
    return result


def blunder_notes(result: EncoderResult) -> list[str]:
    """How the blunder numbers have to be read; the report and the model card share this text."""
    fixed = result.encoder_blunder_fixed
    if result.encoder_blunder is None or fixed is None:
        return []
    if result.threshold_tuned is None or result.threshold_fixed is None:
        return []
    notes = [
        f"the blunder threshold {result.threshold_tuned:.4g} was chosen on the "
        f"'{TUNE_HALF}' half ({result.tune_items:,} rows, {result.tune_games:,} games) by "
        f"maximising F1 there, and the reported numbers are measured on the '{SCORE_HALF}' half "
        f"({result.score_items:,} rows, {result.score_games:,} games)"
        + ("" if result.threshold_split_degenerate else ", which no threshold ever saw")
        + f"; the same rows at the fixed threshold {result.threshold_fixed:.4g} give an F1 of "
        f"{fixed.f1:.4f} against {result.encoder_blunder.f1:.4f} tuned",
        "the material baseline is a hard yes/no rule: it has no threshold, so nothing was tuned "
        "on its side and it got no half to tune on. The margin therefore compares a model at its "
        "best operating point against a rule at its only one",
        base_rate_caveat(result.blunder_base_rate),
    ]
    if result.threshold_split_degenerate:
        notes.append(
            f"the by-game split could not give both halves blunders ({result.blunder_positives} "
            f"of {result.blunder_items} labelled rows are blunders), so the threshold was chosen "
            "and measured on the same rows: this F1 is an upper bound, not a held-out number"
        )
    return notes

def _percent(value: float | None) -> str:
    return "n/a" if value is None else f"{value * 100:.1f} %"


def _number(value: float | None) -> str:
    return "n/a" if value is None else f"{value:.3f}"


def _verdict_cell(met: bool | None) -> str:
    """How an acceptance criterion reads in the report: measured, missed, or not measured."""
    return "not measured" if met is None else ("yes" if met else "no")


def _detector_row(measured: ClassificationResult | None, operating_point: str) -> str | None:
    if measured is None:
        return None
    return (
        f"| {measured.name} | {operating_point} | {measured.items} | {measured.positives} | "
        f"{measured.predicted} | {_percent(measured.precision)} | "
        f"{_percent(measured.recall)} | {_percent(measured.f1)} |"
    )

def _blunder_section(result: EncoderResult) -> list[str]:
    """The whole blunder block: how the operating point was chosen, then every number it gave."""
    tuned = "n/a" if result.threshold_tuned is None else f"{result.threshold_tuned:.4g}"
    fixed = "n/a" if result.threshold_fixed is None else f"{result.threshold_fixed:.4g}"
    lines = [
        "## Blunder detection",
        "",
        sentence(base_rate_caveat(result.blunder_base_rate)),
        "",
        f"The {result.blunder_items:,} labelled rows are cut in two by `game_id`, never by "
        "position, the policy the held-out split itself uses, because two positions of the same "
        f"game are one move apart. The `{TUNE_HALF}` half ({result.tune_items:,} rows, "
        f"{result.tune_games:,} games) chose the threshold `p >= {tuned}` by maximising F1 "
        f"there; the `{SCORE_HALF}` half ({result.score_items:,} rows, {result.score_games:,} "
        "games) is where every number below is measured"
        + ("." if result.threshold_split_degenerate else ", and no `game_id` is in both halves."),
        "",
        "The material baseline is a hard yes/no rule: it has no threshold, so it was given no "
        "half to tune on and nothing was swept on its side. The margin below compares the model "
        "at its best operating point against the rule at its only one, and that is a courtesy "
        "the model receives, not one the baseline does.",
        "",
        "| Detector | Operating point | Items | Blunders | Flagged | Precision | Recall | F1 |",
        "|---|---|---:|---:|---:|---:|---:|---:|",
    ]
    rows = [
        _detector_row(result.encoder_blunder, f"tuned, `p >= {tuned}` (chosen on `{TUNE_HALF}`)"),
        _detector_row(result.encoder_blunder_fixed, f"fixed, `p >= {fixed}`"),
        _detector_row(result.heuristic_blunder, "rule, no threshold to tune"),
    ]
    lines.extend(row for row in rows if row is not None)
    lines.append("")

    if result.tune_f1 is not None:
        lines.extend(
            [
                f"The same threshold reaches an F1 of {_percent(result.tune_f1)} on the "
                f"`{TUNE_HALF}` half it was chosen on. The distance between that and the "
                f"`{SCORE_HALF}` half above is what picking an operating point costs, and it is "
                "the reason the two halves are not the same rows.",
                "",
            ]
        )
    if result.threshold_split_degenerate:
        lines.extend(
            [
                "**Warning:** the by-game split could not give both halves blunders, so the "
                "threshold was chosen and measured on the same rows. The F1 above is an upper "
                "bound, not a held-out number.",
                "",
            ]
        )

    measured = result.encoder_blunder_ranking
    if measured is not None:
        span = "n/a"
        if measured.min_score is not None and measured.max_score is not None:
            span = f"{measured.min_score:.4f} to {measured.max_score:.4f}"
            if measured.mean_score is not None:
                span += f" (mean {measured.mean_score:.4f})"
        lines.extend(
            [
                "### Without an operating point",
                "",
                "ROC AUC is the probability that a random blunder is ranked above a random quiet "
                "move (0.5 is a coin flip); average precision is the area under the "
                "precision-recall curve, and a random ranking scores the base rate, which is "
                "printed next to it. Neither depends on a threshold, so neither can be flattered "
                "by choosing one. The span of the probabilities is here because it is what makes "
                "a fixed threshold reasonable or absurd.",
                "",
                "| Detector | Items | Blunders | Base rate | ROC AUC | Average precision | "
                "Probability span |",
                "|---|---:|---:|---:|---:|---:|---|",
                f"| {measured.name} | {measured.items} | {measured.positives} | "
                f"{_percent(measured.base_rate)} | {_number(measured.roc_auc)} | "
                f"{_number(measured.average_precision)} | {span} |",
                "",
            ]
        )
    return lines

def render_markdown(result: EncoderResult) -> str:
    """The human-readable report: the margin first, then every breakdown."""
    encoder = result.encoder_blunder
    fixed = result.encoder_blunder_fixed
    baseline = result.heuristic_blunder
    measured = result.encoder_blunder_ranking
    margin = "n/a" if result.f1_margin is None else f"{result.f1_margin:+.1f} points"
    target = result.encoder_value.target if result.encoder_value else "tanh(cp / value_scale)"
    tuned = "n/a" if result.threshold_tuned is None else f"{result.threshold_tuned:.4g}"
    at_fixed = "n/a" if result.threshold_fixed is None else f"{result.threshold_fixed:.4g}"
    lines: list[str] = [
        f"# Evaluation of `{result.stage}`",
        "",
        f"- Suite: `{result.suite}`",
        f"- Checkpoint: `{result.checkpoint}`",
        f"- Weights SHA-256: `{result.model_sha}`",
        f"- Parameters: {result.params:,}",
        f"- Device: `{result.device}`",
        f"- Date: {result.date}",
        f"- MLflow run: {result.run_id or 'not tracked'}",
        "",
        "## Headline",
        "",
        "| Metric | Value |",
        "|---|---|",
        f"| Blunder F1, encoder (tuned, `p >= {tuned}`) | "
        f"{_percent(encoder.f1 if encoder else None)} |",
        f"| Blunder F1, encoder (fixed, `p >= {at_fixed}`) | "
        f"{_percent(fixed.f1 if fixed else None)} |",
        f"| Blunder F1, material baseline | {_percent(baseline.f1 if baseline else None)} |",
        f"| Margin over the baseline | {margin} |",
        f"| Margin >= {GOAL_MARGIN:.0f} points | {_verdict_cell(result.meets_goal)} |",
        f"| Blunder ROC AUC | {_number(measured.roc_auc if measured else None)} |",
        f"| Blunder average precision | "
        f"{_number(measured.average_precision if measured else None)} |",
        f"| Blunder base rate | {_percent(result.blunder_base_rate)} |",
        f"| Value vs `{target}`, Spearman (headline) | "
        f"{_number(result.encoder_value.spearman if result.encoder_value else None)} |",
        f"| Value vs `{target}`, Pearson | "
        f"{_number(result.encoder_value.pearson if result.encoder_value else None)} |",
        f"| Value correlation >= {GOAL_VALUE_CORRELATION:.2f} | "
        f"{_verdict_cell(result.value_correlation_meets_goal)} |",
        f"| Result accuracy | {_percent(result.result_accuracy)} |",
        f"| Positions | {result.items:,} ({result.blunder_items:,} with a blunder label, "
        f"{result.score_items:,} of them scored) |",
        "",
    ]

    if encoder is not None or baseline is not None:
        lines.extend(_blunder_section(result))
    if result.encoder_value is not None:
        lines.extend(
            [
                "## Value against Stockfish",
                "",
                f"Both predictors are correlated against `{target}`, the bounded score the value "
                "head is trained on, never against raw `cp`: a forced mate is worth ±9 99x "
                "there and would decide Pearson for the whole set on its own. Spearman asks "
                "only whether the ranking is right and is the number the goal is checked "
                "against; Pearson also asks whether the scale is.",
                "",
                "| Predictor | Items | Target | Spearman | Pearson |",
                "|---|---:|---|---:|---:|",
            ]
        )
        for measured in (result.encoder_value, result.heuristic_value):
            if measured is not None:
                lines.append(
                    f"| {measured.name} | {measured.items} | `{measured.target}` | "
                    f"{_number(measured.spearman)} | {_number(measured.pearson)} |"
                )
        lines.append("")
    if result.label_curve:
        keys = sorted({key for point in result.label_curve for key in point.metrics})
        lines.extend(
            [
                "## Labels needed",
                "",
                "| Labels | Rows | " + " | ".join(f"`{key}`" for key in keys) + " |",
                "|---:|---:|" + "---:|" * len(keys),
            ]
        )
        for point in result.label_curve:
            cells = " | ".join(_number(point.metrics.get(key)) for key in keys)
            rows = "n/a" if point.train_labels is None else f"{point.train_labels:,}"
            lines.append(f"| {point.fraction:.0%} | {rows} | {cells} |")
        lines.append("")
    if result.notes:
        lines.extend(["## Notes", ""])
        lines.extend(f"- {note}" for note in result.notes)
        lines.append("")
    return "\n".join(lines)

def track_result(result: EncoderResult, cfg: EncoderEvalConfig) -> str | None:
    """Log the suite to the local MLflow store and return the run id."""
    import mlflow

    from rukh.tracking import start_run

    with start_run(
        f"eval-{result.stage}",
        {"suite": result.suite, "checkpoint": result.checkpoint, **cfg.model_dump(mode="json")},
        tags={"kind": "eval", "stage": result.stage, "model_sha": result.model_sha},
    ) as run:
        metrics = _metrics(result)
        if metrics:
            mlflow.log_metrics(metrics)
        return str(run.info.run_id)

def _metrics(result: EncoderResult) -> dict[str, float]:
    """Flat metrics for MLflow (only what was actually measured)."""
    metrics: dict[str, float] = {}
    for measured, name in (
        (result.encoder_blunder, "encoder"),
        (result.encoder_blunder_fixed, "encoder_fixed"),
        (result.heuristic_blunder, "heuristic"),
    ):
        if measured is not None:
            metrics[f"blunder_f1/{name}"] = measured.f1
            metrics[f"blunder_precision/{name}"] = measured.precision
            metrics[f"blunder_recall/{name}"] = measured.recall
    if result.encoder_blunder_ranking is not None:
        auc = result.encoder_blunder_ranking.roc_auc
        precision = result.encoder_blunder_ranking.average_precision
        if auc is not None:
            metrics["blunder_roc_auc"] = auc
        if precision is not None:
            metrics["blunder_average_precision"] = precision
    for name, value in (
        ("blunder_threshold", result.threshold_tuned),
        ("blunder_base_rate", result.blunder_base_rate),
        ("blunder_f1_tune", result.tune_f1),
    ):
        if value is not None:
            metrics[name] = float(value)
    for measured in (result.encoder_value, result.heuristic_value):
        if measured is not None and measured.pearson is not None:
            metrics[f"value_pearson/{measured.name}"] = measured.pearson
        if measured is not None and measured.spearman is not None:
            metrics[f"value_spearman/{measured.name}"] = measured.spearman
    if result.f1_margin is not None:
        metrics["blunder_f1_margin"] = result.f1_margin
    for name, met in (
        ("meets_goal_blunder", result.meets_goal),
        ("meets_goal_value", result.value_correlation_meets_goal),
        ("meets_goal_all", result.meets_all_goals),
    ):
        if met is not None:
            metrics[name] = float(met)
    if result.result_accuracy is not None:
        metrics["result_accuracy"] = result.result_accuracy
    return metrics