GRPO y la recompensa que un motor verifica

from __future__ import annotations

import chess
from pydantic import Field

from rukh.config import BaseConfig

__all__ = [
    "ILLEGAL_MARGIN",
    "RewardBreakdown",
    "RewardWeights",
    "illegal_value",
    "legal_bounds",
    "legality_gate",
    "mate_bonus",
    "normalised_delta_cp",
    "reward",
    "repetition_penalty",
]

"""The reward functions, written before anything trains on them.

A learned reward can be wrong in ways nobody notices. A verifiable one can only be **designed**
wrong, and a design is testable: the best move gets the most, an illegal move gets nothing, and no
input anywhere produces a number outside the interval the design promises.
"""

from __future__ import annotations

import random

import chess
import pytest

from rukh.train.rewards import (
    RewardWeights,
    bounds,
    illegal_value,
    legal_bounds,
    legality_gate,
    mate_bonus,
    normalised_delta_cp,
    repetition_penalty,
    reward,
)

pytestmark = pytest.mark.unit


def test_legality_is_a_gate_and_not_a_term():
    """An illegal move scores the floor flat, with no other term able to buy it back."""
    board = chess.Board()
    illegal = chess.Move.from_uci("e2e5")
    assert legality_gate(board, illegal) is False

    got = reward(board, illegal, cp_move=1000, cp_best=0)
    assert got.total == illegal_value()
    assert got.legal is False
    # Even with the best possible centipawn score behind it, nothing leaks through the gate.
    assert got.terms == {"quality": 0.0, "mate": 0.0, "repetition": 0.0}


def test_an_illegal_move_scores_below_every_legal_one():
    """The bug the M5 gallery found: a flat zero put illegal *above* a legal repetition.

    With the default weights a legal move that repeats scores -0.25, so a gate set at 0.0 was
    telling the optimiser that proposing a move that cannot be played beats playing a legal one.
    """
    for weights in (
        RewardWeights(),
        RewardWeights(repetition=2.0),
        RewardWeights(quality=3.0, mate=1.0, repetition=0.5),
    ):
        assert illegal_value(weights) < legal_bounds(weights)[0]
        assert bounds(weights) == (illegal_value(weights), legal_bounds(weights)[1])

def test_the_best_move_scores_one_and_the_scale_is_where_it_runs_out():
    assert normalised_delta_cp(50, 50) == 1.0
    assert normalised_delta_cp(-150, 50, scale=200) == 0.0
    assert normalised_delta_cp(-50, 50, scale=200) == pytest.approx(0.5)
    # Better than the best is not a thing; the clamp says so instead of producing 1.2.
    assert normalised_delta_cp(100, 50, scale=200) == 1.0
    # And a catastrophe is not more informative than a loss: both bottom out.
    assert normalised_delta_cp(-100_000, 50, scale=200) == 0.0


def test_a_scale_of_zero_is_refused_rather_than_dividing():
    with pytest.raises(ValueError, match="scale must be positive"):
        normalised_delta_cp(0, 100, scale=0)


def test_mate_is_binary_and_does_not_pay_for_being_quick():
    """Fool's mate: `Qh5xf7#` ends it, and a shorter mate is the same outcome sooner."""
    board = chess.Board()
    for uci in ("f2f3", "e7e5", "g2g4"):
        board.push_uci(uci)
    mate = chess.Move.from_uci("d8h4")
    assert mate_bonus(board, mate) == 1.0
    assert mate_bonus(board, chess.Move.from_uci("g8f6")) == 0.0


def test_the_repetition_penalty_fires_on_the_second_visit_not_on_the_claim():
    """By the time a draw is claimable the shuffling already happened; the penalty is too late."""
    board = chess.Board()
    for uci in ("g1f3", "g8f6", "f3g1", "f6g8"):
        board.push_uci(uci)
    # The starting position is on the board again, so `Nf3` returns to a position already seen.
    assert repetition_penalty(board, chess.Move.from_uci("g1f3")) == 1.0
    assert repetition_penalty(board, chess.Move.from_uci("e2e4")) == 0.0


def test_the_reward_carries_its_terms_because_a_scalar_hides_its_design():
    board = chess.Board()
    got = reward(board, chess.Move.from_uci("e2e4"), cp_move=20, cp_best=30)
    assert got.legal is True
    assert got.quality == pytest.approx(1.0 - 10 / 200)
    assert got.total == pytest.approx(got.quality)
    assert set(got.terms) == {"quality", "mate", "repetition"}


def test_without_engine_scores_the_reward_falls_back_to_what_is_knowable():
    """No `cp` means no quality term -- not a guessed one."""
    board = chess.Board()
    got = reward(board, chess.Move.from_uci("e2e4"))
    assert got.legal is True
    assert got.quality == 0.0
    assert got.total == 0.0

def test_no_input_escapes_the_interval_the_design_promises():
    """A reward with no ceiling is an invitation; this is the test that keeps the promise.

    Random legal moves from random positions, with centipawn scores drawn far outside anything an
    engine would return, including the mate scores the pipeline uses.
    """
    weights = RewardWeights()
    low, high = bounds(weights)
    rng = random.Random(7)
    seen = 0
    for _ in range(300):
        board = chess.Board()
        for _ in range(rng.randrange(0, 40)):
            moves = list(board.legal_moves)
            if not moves:
                break
            board.push(rng.choice(moves))
        moves = list(board.legal_moves)
        if not moves:
            continue
        move = rng.choice(moves)
        cp_move = rng.choice([-10_000, -999, 0, 37, 10_000])
        cp_best = rng.choice([-10_000, -12, 0, 500, 10_000])
        total = reward(board, move, cp_move, cp_best, weights).total
        assert low <= total <= high, (board.fen(), move.uci(), cp_move, cp_best, total)
        seen += 1
    assert seen > 200, "the walk should have produced plenty of legal positions"


def test_the_bounds_move_with_the_weights_instead_of_being_a_constant():
    weights = RewardWeights(quality=2.0, mate=0.5, repetition=1.0)
    assert legal_bounds(weights) == (-1.0, 2.5)
    assert bounds(weights) == (-2.0, 2.5)


def test_the_weights_are_refused_when_they_would_break_the_design():
    with pytest.raises(ValueError):
        RewardWeights(quality=-1.0)
    with pytest.raises(ValueError):
        RewardWeights(cp_scale=0)

from __future__ import annotations

import json
import logging
import math
from typing import TYPE_CHECKING

import chess
import chess.engine
import numpy as np
import torch
import torch.nn.functional as F
from pydantic import Field

from rukh.config import BaseConfig
from rukh.tokenize.uci_vocab import UciTokenizer, elo_token
from rukh.train.rewards import RewardWeights, illegal_value, reward

if TYPE_CHECKING:
    from collections.abc import Sequence
    from pathlib import Path

LOGGER = logging.getLogger(__name__)

__all__ = [
    "EngineCache",
    "best_available",
    "GrpoConfig",
    "GrpoResult",
    "group_advantages",
    "grpo_loss",
    "sample_group",
    "train_grpo",
]

class GrpoConfig(BaseConfig):
    """The prompts, the policy, the size of a group and the price of scoring one."""

    prompts: str = "data/pairs/dpo-prompts.parquet"
    """Only the ``prefix`` column is read. GRPO needs positions, not preferences -- which is the
    point worth noticing: the same file feeds both methods and only one of them uses the labels."""
    checkpoint: str = ""
    out_dir: str = "checkpoints"
    run_name: str = "grpo"
    group_size: int = Field(default=8, ge=2)
    """Candidates per position. The baseline is their mean, so two is the minimum that has one."""
    temperature: float = Field(default=1.0, gt=0)
    top_k: int | None = 20
    """How many candidate tokens the group is drawn from, after the restriction below."""
    restrict_to_legal: bool = True
    """Whether the group is drawn only from moves that can be played here.

    ``True`` spends the whole engine budget on moves the model could actually play, and the
    legality gate then never fires -- which is fine, because the demo masks illegal moves before
    they reach the board, so legality is a property the reward *guarantees* rather than one it
    teaches.

    ``False`` draws from the whole vocabulary, so an illegal proposal can land in the group and
    score the floor. That is the only configuration in which GRPO can teach legality at all, and
    M5 needed it: DPO doubled the illegal-proposal rate of its base (D-121), and a reward whose
    first rule is a legality gate ought to be able to pay that back."""
    beta_kl: float = Field(default=0.02, ge=0)
    """Weight of the KL term against the frozen starting policy."""
    lr: float = Field(default=1e-6, gt=0)
    positions_per_step: int = Field(default=8, ge=1)
    steps: int = Field(default=200, ge=1)
    depth: int = Field(default=8, ge=1)
    """Engine depth per candidate. Depth, not time, so a run is reproducible (D-107)."""
    block: int = Field(default=200, ge=8)
    weights: RewardWeights = RewardWeights()
    cache: str | None = "artifacts/grpo-engine-cache.json"
    """Where the ``(fen, move) -> centipawns`` cache lives between runs. ``None`` disables it."""
    val_positions: int = Field(default=200, ge=0)
    """Held-out positions the mean reward is measured on, before and after."""
    seed: int = 42
    device: str | None = None

class GrpoResult(BaseConfig):
    """What the run produced, in the shape the lesson quotes it."""

    checkpoint: str
    steps: int
    groups: int
    """Groups that contributed a gradient."""
    flat_groups: int
    """Groups where every candidate scored the same, so the baseline ate the whole signal."""
    engine_calls: int
    cache_hits: int
    reward_before: float
    reward_after: float
    """Mean reward with ``cp_best`` taken **inside the group**, which is what training optimises.

    Read it as a training diagnostic and never as a score. It is maximised by a policy that
    proposes the same move eight times out of eight: then the best candidate *is* every candidate,
    every quality term is 1.0, and the mean is as high as it goes. A model that collapses makes
    this number go up. ``reward_absolute_*`` is the one that cannot be gamed that way."""
    reward_absolute_before: float = math.nan
    reward_absolute_after: float = math.nan
    """Mean reward with ``cp_best`` taken from the **engine's own best move** in the position.

    One extra analysis per position, not per candidate, and it is the number that says whether the
    moves got better. Collapsing the policy does not move it: if the group agrees on a mediocre
    move, all eight are still mediocre against what was available."""
    flat_share_before: float = math.nan
    flat_share_after: float = math.nan
    """Share of validation groups whose candidates were all the same move. The collapse detector."""
    kl_after: float
    """Mean KL over the last tenth of the run, not the last batch.

    It was the last batch until a sweep reported 0.00048 and 0.126 for the same learning rate on
    two runs that differed in nothing that mattered: one group's KL is a sample, not a level."""
    illegal_before: float
    illegal_after: float

class EngineCache:
    """``(fen, move) -> centipawns for the mover``, kept between steps and between runs.

    Keyed by the FEN rather than by the move list, so a position reached by transposition is the
    same key. That is not a micro-optimisation: with eight candidates per position and two hundred
    steps, the engine is the entire cost of the run, and the policy keeps proposing the moves it
    already likes.
    """

    def __init__(self, path: Path | None = None) -> None:
        self.path = path
        self.data: dict[str, int] = {}
        self.hits = 0
        self.calls = 0
        if path is not None and path.exists():
            self.data = json.loads(path.read_text("utf-8"))
            LOGGER.info("cache: %d analisis reutilizables en %s", len(self.data), path)

    @staticmethod
    def key(fen: str, move: chess.Move) -> str:
        return f"{fen}|{move.uci()}"

    def score(
        self, engine: chess.engine.SimpleEngine, board: chess.Board, move: chess.Move, depth: int
    ) -> int:
        """Centipawns for the side that plays ``move``, from the cache when it is there."""
        from rukh.data.onpolicy import score_move

        key = self.key(board.fen(), move)
        cached = self.data.get(key)
        if cached is not None:
            self.hits += 1
            return cached
        value = score_move(engine, board, move, depth)
        self.data[key] = value
        self.calls += 1
        return value

    def save(self) -> None:
        if self.path is None:
            return
        self.path.parent.mkdir(parents=True, exist_ok=True)
        self.path.write_text(json.dumps(self.data), "utf-8")

def _prompt(tok: UciTokenizer, prefix: str, white: int, black: int, block: int) -> list[int] | None:
    """The same three header tokens every other measurement uses, then the game so far."""
    ids = [tok.bos_id, tok.vocab[elo_token(white, "w")], tok.vocab[elo_token(black, "b")]]
    for uci in prefix.split():
        token = tok.vocab.get(uci)
        if token is None:
            return None
        ids.append(int(token))
    return ids if len(ids) <= block else None


def _board_for(prefix: str) -> chess.Board | None:
    board = chess.Board()
    try:
        for uci in prefix.split():
            board.push_uci(uci)
    except ValueError:
        return None
    return None if board.is_game_over() else board


def _move_of(tok: UciTokenizer, token: int) -> chess.Move | None:
    """The move a token spells, or ``None`` when the token is not a move at all.

    The reverse table is built once per tokenizer and cached on it, because this runs eight times
    per position and rebuilding a 2 030-entry dict each time would be the second cost of the run
    after the engine.
    """
    inverse = getattr(tok, "_rukh_inverse", None)
    if inverse is None:
        inverse = {index: text for text, index in tok.vocab.items()}
        tok._rukh_inverse = inverse  # type: ignore[attr-defined]
    text = inverse.get(token)
    if text is None or len(text) < 4:
        return None
    try:
        return chess.Move.from_uci(text)
    except ValueError:
        return None

def train_grpo(cfg: GrpoConfig) -> GrpoResult:
    """Run GRPO and return what it measured, including the two numbers the lesson needs."""
    import copy

    import polars as pl

    from rukh import paths
    from rukh.engine import find_stockfish
    from rukh.infer.sampler import legal_token_ids
    from rukh.train.checkpoint import load_model, save_checkpoint
    from rukh.train.common import pick_device

    binary = find_stockfish()
    if binary is None:
        raise FileNotFoundError("Stockfish not found; set RUKH_STOCKFISH")
    torch.manual_seed(cfg.seed)
    device = torch.device(cfg.device or pick_device())
    tok = UciTokenizer()

    policy, payload = load_model(paths.resolve(cfg.checkpoint), device)
    policy = policy.to(device).train()
    reference = copy.deepcopy(policy).eval()
    for parameter in reference.parameters():
        parameter.requires_grad_(False)

    frame = pl.read_parquet(paths.resolve(cfg.prompts).as_posix())
    rng = np.random.default_rng(cfg.seed)
    order = rng.permutation(frame.height)
    val_index = order[: cfg.val_positions]
    train_index = order[cfg.val_positions :]

    cache = EngineCache(paths.resolve(cfg.cache) if cfg.cache else None)
    optimizer = torch.optim.AdamW(policy.parameters(), lr=cfg.lr)
    generator = torch.Generator(device=device).manual_seed(cfg.seed)

    def group_at(row: dict[str, object]) -> tuple[chess.Board, list[int], torch.Tensor] | None:
        """Prompt, board and a sampled group for one row, or ``None`` when the row is unusable."""
        board = _board_for(str(row["prefix"] or ""))
        ids = _prompt(
            tok, str(row["prefix"] or ""), int(row["white_elo"]), int(row["black_elo"]), cfg.block
        )
        if board is None or ids is None:
            return None
        with torch.no_grad():
            logits, _ = policy(torch.tensor([ids], dtype=torch.long, device=device))
        tokens = sample_group(
            logits[0, -1],
            legal_token_ids(board, tok) if cfg.restrict_to_legal else None,
            cfg.group_size,
            cfg.temperature,
            cfg.top_k,
            generator,
        )
        return board, ids, tokens

    cache.save()

    out_dir = paths.resolve(cfg.out_dir) / cfg.run_name
    out_dir.mkdir(parents=True, exist_ok=True)
    final = out_dir / "grpo.pt"
    save_checkpoint(
        final,
        step=cfg.steps,
        model=policy,
        optimizer=None,
        cfg={**cfg.model_dump(mode="json"), "reward_before": before[0], "reward_after": after[0]},
        model_cfg=payload["model_cfg"],
        vocab_hash=payload.get("vocab_hash"),
        data_manifest_sha=payload.get("data_manifest_sha"),
    )
    return GrpoResult(
        checkpoint=str(final),
        steps=cfg.steps,
        groups=groups,
        flat_groups=flat,
        engine_calls=cache.calls,
        cache_hits=cache.hits,
        reward_before=before[0],
        reward_after=after[0],
        reward_absolute_before=before[1],
        reward_absolute_after=after[1],
        flat_share_before=before[3],
        flat_share_after=after[3],
        kl_after=last_kl,
        illegal_before=before[2],
        illegal_after=after[2],
    )

"""GRPO: the baseline, the penalty, and the two ways this loop can lie to itself.

The parts worth testing here are the ones where a bug produces a *number* rather than a crash: a
baseline that manufactures a gradient out of a group that all agreed, a KL term with the wrong
sign that pays the model for leaving its reference, and a cache that answers for the wrong
position.
"""

from __future__ import annotations

import chess
import pytest
import torch

from rukh.train.grpo import (
    EngineCache,
    GrpoConfig,
    group_advantages,
    grpo_loss,
    sample_group,
    score_candidates,
)

pytestmark = pytest.mark.unit


def test_a_group_that_agrees_produces_no_gradient():
    """The baseline is the group's own mean, so a flat group has nothing left to say.

    Dividing by a spread of zero would hand back whatever the epsilon makes of floating-point
    dust, and the loop would spend a step chasing it.
    """
    assert torch.equal(group_advantages([0.42] * 8), torch.zeros(8))
    assert torch.equal(group_advantages([0.0, 0.0]), torch.zeros(2))


def test_the_advantages_are_centred_and_scaled_by_the_groups_own_spread():
    advantages = group_advantages([1.0, 0.0, 0.5, 0.5])
    assert float(advantages.mean()) == pytest.approx(0.0, abs=1e-5)
    assert float(advantages[0]) > 0 and float(advantages[1]) < 0
    # A group ten times as spread out gets the same advantages: only the ordering is kept.
    wide = group_advantages([10.0, 0.0, 5.0, 5.0])
    assert torch.allclose(advantages, wide, atol=1e-3)

def test_the_kl_term_is_never_negative():
    """k3 is non-negative by construction, which is the point of using it.

    The naive ``logp_ref - logp_policy`` is negative on exactly the samples where the policy moved
    away from the reference, so as a *penalty* it would pay for the behaviour it is meant to
    discourage. The test draws both directions on purpose.
    """
    torch.manual_seed(0)
    actions = torch.tensor([0, 1, 2, 3])
    advantages = torch.zeros(4)
    for _ in range(20):
        policy = torch.log_softmax(torch.randn(4, 8), dim=-1)
        reference = torch.log_softmax(torch.randn(4, 8), dim=-1)
        _loss, kl = grpo_loss(policy, reference, actions, advantages, beta_kl=1.0)
        assert float(kl) >= 0.0


def test_the_kl_is_zero_when_the_policy_is_still_the_reference():
    """The first step of any run: the penalty must cost nothing before anything has moved."""
    logprobs = torch.log_softmax(torch.randn(4, 8), dim=-1)
    _loss, kl = grpo_loss(
        logprobs, logprobs.clone(), torch.tensor([0, 1, 2, 3]), torch.zeros(4), beta_kl=1.0
    )
    assert float(kl) == pytest.approx(0.0, abs=1e-6)


def test_a_positive_advantage_pushes_its_move_up():
    """The smallest end-to-end check of the sign: one step of the loss raises the chosen move."""
    torch.manual_seed(0)
    logits = torch.zeros(1, 4, requires_grad=True)
    actions = torch.tensor([2])
    advantages = torch.tensor([1.0])
    for _ in range(50):
        logprobs = torch.log_softmax(logits, dim=-1)
        loss, _kl = grpo_loss(logprobs, logprobs.detach(), actions, advantages, beta_kl=0.0)
        loss.backward()
        with torch.no_grad():
            logits -= 0.5 * logits.grad
        logits.grad = None
    assert int(logits.argmax()) == 2

    # And a negative advantage pushes it down, which is the same test with the sign flipped.
    logits = torch.zeros(1, 4, requires_grad=True)
    for _ in range(50):
        logprobs = torch.log_softmax(logits, dim=-1)
        loss, _kl = grpo_loss(logprobs, logprobs.detach(), actions, -advantages, beta_kl=0.0)
        loss.backward()
        with torch.no_grad():
            logits -= 0.5 * logits.grad
        logits.grad = None
    assert int(logits.argmin()) == 2

def test_sampling_only_ever_draws_from_the_moves_it_was_given():
    """Everything outside ``legal_ids`` is masked to minus infinity, however large its logit."""
    logits = torch.zeros(10)
    logits[7] = 50.0  # the model is very sure about a move that is not on the list
    drawn = sample_group(logits, [1, 2, 3], size=64, temperature=1.0, top_k=None)
    assert set(int(token) for token in drawn) <= {1, 2, 3}


def test_sampling_keeps_duplicates_because_the_group_is_a_sample_of_the_policy():
    """A policy that would play one move eight times out of eight is saying something true."""
    logits = torch.zeros(10)
    logits[2] = 30.0
    drawn = sample_group(logits, [1, 2, 3], size=8, temperature=1.0, top_k=None)
    assert drawn.shape == (8,)
    assert set(int(token) for token in drawn) == {2}


class _CountingEngine:
    """An engine that refuses to be called twice for the same thing."""

    def __init__(self) -> None:
        self.seen: list[str] = []

    def analyse(self, board: chess.Board, limit: object) -> dict[str, object]:
        self.seen.append(board.fen())
        return {"score": chess.engine.PovScore(chess.engine.Cp(33), chess.WHITE)}


def test_the_cache_answers_the_second_time_without_the_engine():
    cache = EngineCache(None)
    engine = _CountingEngine()
    board = chess.Board()
    move = chess.Move.from_uci("e2e4")
    first = cache.score(engine, board, move, depth=4)
    second = cache.score(engine, board, move, depth=4)
    assert first == second == 33
    assert len(engine.seen) == 1
    assert (cache.calls, cache.hits) == (1, 1)


def test_the_cache_is_keyed_by_the_position_not_by_the_move_list():
    """Two move orders reaching the same position are one cache entry, which is why it pays."""
    direct = chess.Board()
    direct.push_uci("g1f3")
    direct.push_uci("g8f6")
    transposed = chess.Board()
    transposed.push_uci("g1f3")
    transposed.push_uci("g8f6")
    move = chess.Move.from_uci("d2d4")
    assert EngineCache.key(direct.fen(), move) == EngineCache.key(transposed.fen(), move)


def test_the_config_refuses_a_group_that_cannot_have_a_baseline():
    with pytest.raises(ValueError):
        GrpoConfig(group_size=1)
    with pytest.raises(ValueError):
        GrpoConfig(beta_kl=-0.1)

def test_a_collapsed_group_scores_the_maximum_against_its_own_best():
    """The flaw that makes the training reward useless as a score, pinned so it stays documented.

    ``score_candidates`` takes ``cp_best`` over the candidates themselves, which is what keeps a
    group of bad moves teachable -- the ordering inside it survives. The price is that a policy
    proposing one move eight times gets quality 1.0 on all eight, which is the highest the reward
    goes. So the training number is maximised by collapsing, and only the version that takes
    ``cp_best`` from the engine can be read as a score.
    """

    class _Fixed:
        """An engine that says every move leaves the position at -50 for the mover."""

        def analyse(self, board: chess.Board, limit: object) -> dict[str, object]:
            return {"score": chess.engine.PovScore(chess.engine.Cp(50), chess.WHITE)}

    cache = EngineCache(None)
    board = chess.Board()
    move = chess.Move.from_uci("a2a3")  # a legal move, and not a good one
    collapsed = [move] * 8

    relative = score_candidates(cache, _Fixed(), board, collapsed, GrpoConfig())
    assert relative == [pytest.approx(1.0)] * 8, "the group is its own best, so every one is best"

    # Against a genuinely better reference the same eight moves score the floor of the quality
    # term, which is what a mediocre move deserves and what the published number has to use.
    absolute = score_candidates(cache, _Fixed(), board, collapsed, GrpoConfig(), best=400.0)
    assert absolute == [pytest.approx(0.0)] * 8

# GRPO against a verifiable reward. No preference pairs and no reward model: only the positions
# are read out of the pairs file, and the labels in it are ignored.
prompts: data/pairs/dpo-prompts.parquet
checkpoint: checkpoints/medium-v4-20260919-174623/best.pt
out_dir: checkpoints
run_name: medium-v4-grpo

# Eight candidates per position, which is what makes the group's mean a usable baseline. Their
# spread is the entire learning signal, so this is the knob that decides how much of the engine
# budget turns into a gradient and how much is spent on groups that all agree.
group_size: 8

# Temperature 1.0, and the measurement that says this is not a free lever. Raising it buys
# diversity -- 38 % of groups are flat at 1.0, 26 % at 1.3, 19 % at 1.6 -- and charges for it in
# engine calls, so the price per *useful* group barely moves: 0.184, 0.186 and 0.174 groups per
# analysis. With the cost flat, the tie-break is on-policy-ness, and 1.0 is closest to how the
# model is read.
temperature: 1.0
top_k: 20

# Only moves that can be played here. The legality gate then never fires, which is fine: the demo
# masks illegal moves before they reach the board. Set it to false to let the gate teach --
# `grpo-legality.yaml` is the run that does.
restrict_to_legal: true

# The KL term against the frozen start. The reward is verifiable, which means it is exactly
# specifiable and therefore exactly gameable: this is what stops the policy from walking off to
# whatever corner of move-space scores best.
beta_kl: 0.02

lr: 1.0e-6
positions_per_step: 8
steps: 300

# Depth 8 per candidate, and a persistent cache so the second visit to a position is free.
# The engine is the whole cost of this run; the model is almost an afterthought.
depth: 8
cache: artifacts/grpo-engine-cache.json

val_positions: 200

# The reward's weights, written down so the run can be rebuilt from its config alone.
weights:
  quality: 1.0
  mate: 0.25
  repetition: 0.25
  cp_scale: 200.0

seed: 42