# NEXIDO -- INDEPENDENT REPRODUCTION OF A TRACK B (LLM FINE-TUNE) SCORE
#
# Re-scores one scoring stage of your training run on your own machine, with the public model and the scorer we
# run, and compares what it gets with the score.json we gave you. You need, from the job page:
#   the scoring stage's verification record (record.json) and its score.json,
#   the training stage's adapter (the .safetensors file),
# and your own training file, the public Qwen3-4B-Instruct-2507 at the commit below, and these five files side by
# side: reproduce_llm.py (this), score_llm.py, llm_rows.py, lora_file.py and split_rule.py.
#
#     python3 reproduce_llm.py --record record.json --score score.json --adapter adapter.safetensors \
#                              --input your-file.jsonl --model ./Qwen3-4B-Instruct-2507 [--device cpu]
#
# WHAT IT CHECKS BEFORE IT COMPUTES ANYTHING (any failure stops it there):
#   the record is of a scoring stage, and names the scoring image this verifier is for (SCORE_IMAGE);
#   the score.json you kept is the output the record names (its SHA-256 is the output_hash of the record and of its
#     signed receipt: run verify-record.mjs on the same record for the signature itself);
#   the score.json names the part, split, counted file and adapter the record names; every sum in it is a finite
#     number, each decision is the sign of its own two sums, its smallest margin is the smallest of its own, and its
#     decision_sha256 is the digest of its own decisions;
#   the adapter, your file and every file of the model are the ones named, by hash;
#   the four scorer files beside this script are byte for byte the ones inside that scoring image; they are copied,
#     so checked, into a fresh folder, and run from there with no compiled copy of anything.
# THEN it runs the scorer exactly as a node runs it (score_llm.py, one process, the job built from the record, the
# image's own settings: CUBLAS_WORKSPACE_CONFIG=:4096:8, which PyTorch's deterministic mode needs on a GPU,
# PYTHONHASHSEED=0, offline) and compares, conversation by conversation:
#   REPRODUCED BIT FOR BIT  the same score.json, byte for byte. Expected with the pinned stack (STACK, Python 3.10)
#                           on an RTX A4500, the GPU class our scoring runs on (Finding 145: two machines, eight runs,
#                           one hash);
#   REPRODUCED              everything the same but the sums' last digits: the same decision on every conversation,
#                           so the same decision_sha256, every conversation's sum within REL_TOL of ours
#                           (relatively, or absolutely for a sum smaller than one) and both losses within REL_TOL,
#                           relatively. Expected on another GPU or a CPU: an RTX A5000 differed by at most 3.8 parts
#                           in a million;
#   NOT REPRODUCED          anything else, with what differs.
# Exit 0 only for the first two; 3 for either of them with --unpinned-model, which is a test of this script only.
#
# PROVE THE CHECK CAN FAIL. Each control changes ONE thing after the checks above have passed, and must be caught by
# the check named for it, for that very change (other checks may fail too); the run says CONTROL CAUGHT (exit 5), or
# CONTROL NOT CAUGHT (exit 4):
#   BREAK=flip     one decision changed in your copy, its digest made consistent  -> that one conversation differs
#   BREAK=sum      one sum in your copy moved by a thousandth (at least 0.001)   -> that sum is outside the tolerance
#   BREAK=adapter  the adapter's B matrices zeroed, a valid file otherwise        -> "the same decision ..."
#   BREAK=seed     the split's seed changed in the job the scorer is given        -> the scorer refuses the split
#   BREAK=file     the last row dropped from the file the scorer is given         -> the scorer refuses the file
#
# It never contacts Nexido. The scorer needs torch, transformers, peft, safetensors and tokenizers; the versions in
# STACK reproduce our bits, others should reproduce our decisions. A GPU needs 11 GiB free; --device cpu works
# anywhere with about 16 GB of memory, slowly (on a CPU the scorer uses every core, where the image uses one). On
# Windows, run it under WSL.
import argparse, hashlib, io, json, math, os, shutil, subprocess, sys, tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
# NOTHING IS IMPORTED FROM THIS FOLDER: a stray torch.py or json.py beside these files could stand in for the real one.
# The scorer's own modules are imported only from the fresh folder of their checked copies (check_scorer).
sys.path[:] = [p for p in sys.path if os.path.abspath(p or os.curdir) != HERE]
REPO, REV = "Qwen/Qwen3-4B-Instruct-2507", "cdbee75f17c01a7cc42f958dc650907174af0554"
RECORD_VERSION, SCORE_KIND, PLAN_VERSION = "nexido-verification-record-v1", "nexido-score-llm-v1", 1
REL_TOL = 1e-4
RECEIPT_KEYS = ("version", "job_id", "row", "tokenizer_sha256", "render", "limits", "counter_revision",
                "file_sha256", "bytes", "rows", "tokens", "row_tokens_sha256")
SPLIT_KEYS = ("rule_version", "seed", "test_bps", "val_bps", "n_train", "n_val", "n_test", "digest_train",
              "digest_val", "digest_test")
# THE SCORING IMAGE this verifier is for, score-llm v1 (ADR-0067, built by block 123 from commit 98435b3), and the
# scorer inside it: these four files, byte for byte.
SCORE_IMAGE = "sha256:860369248ba8ee9450559677a64281ad661c68301452a070f4d42245058b264b"
SCORER = {"score_llm.py": "ef0bd4ca32253759d77a075845e147106b2dad15b9e37192b5bd574ef55fd339",
          "llm_rows.py": "10cc6f8c3cbf841bdac4efe826c82e9984467e149bb6452cebf42e7680852800",
          "lora_file.py": "e949b5ade8065c5679baff89511572187ab852fd2492d8030da0181cb4eef2ad",
          "split_rule.py": "afa0d68826846b4679289c241af5b4766d839c474b25e0f7e94aa8825d8e0898"}
# THE MODEL, as Hugging Face lists the commit: name, bytes, and its hash there. The three weight files are stored
# by Git LFS and listed by SHA-256; the others by their Git object id (SHA-1 of "blob <size>\0" and the bytes).
# Download it with:  huggingface-cli download Qwen/Qwen3-4B-Instruct-2507 --revision <REV> --local-dir <folder>
MODEL_FILES = (
    (".gitattributes", 1570, "52373fe24473b1aa44333d318f578ae6bf04b49b"),
    ("LICENSE", 11343, "6634c8cc3133b3848ec74b9f275acaaa1ea618ab"),
    ("README.md", 8168, "b85879f8e465b5b02bd7ea13dd7ffaf22816bf59"),
    ("config.json", 727, "6988f134db143052042f2bd6e0c897bc6a605189"),
    ("generation_config.json", 238, "432531a002c181a19de338313d2375e9d7494d7e"),
    ("merges.txt", 1671839, "20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0"),
    ("model-00001-of-00003.safetensors", 3957900840,
     "75311d91bb08cf0b882913da464a1e722a31fb44db35208663487efb7a3d8ed6"),
    ("model-00002-of-00003.safetensors", 3987450520,
     "0b48adbb1f60e901153d91907ba11ce63bd4b8b584482e730f48808d055dfba1"),
    ("model-00003-of-00003.safetensors", 99630640,
     "7dd39ccca5e4de123c74c14af44c9bf2eb75df33b4614382af0134528e060d5d"),
    ("model.safetensors.index.json", 32819, "4747b0297d3109f14db49886972e3369c9a00b2a"),
    ("tokenizer.json", 11422654, "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4"),
    ("tokenizer_config.json", 9377, "51c1be0d9192e7f6e6596de71d0f07d58fbc32ac"),
    ("vocab.json", 2776833, "4783fe10ac3adce15ac8f358ef5462739852c569"))
# THE STACK OUR IMAGE RUNS (Python 3.10.12): the same versions are what make the same bits possible.
STACK = (("torch", "2.4.1"), ("transformers", "4.57.6"), ("peft", "0.20.0"), ("safetensors", "0.8.0"),
         ("tokenizers", "0.22.2"))
# THE SCORING IMAGE'S OWN SETTINGS, as the scorer runs under them (agent/eval/Dockerfile, inherited): deterministic
# cuBLAS, which PyTorch's deterministic mode refuses a GPU without; a fixed hash seed; nothing fetched. On a GPU also
# its one thread; on a CPU every core, or the re-score would take many hours.
SCORER_ENV = {"CUBLAS_WORKSPACE_CONFIG": ":4096:8", "PYTHONHASHSEED": "0", "PYTHONDONTWRITEBYTECODE": "1",
              "TOKENIZERS_PARALLELISM": "false", "HF_HUB_OFFLINE": "1", "HF_DATASETS_OFFLINE": "1",
              "TRANSFORMERS_OFFLINE": "1", "HF_HUB_DISABLE_TELEMETRY": "1",
              "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"}
GPU_ENV = {"OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"}
BREAKS = ("flip", "sum", "adapter", "seed", "file")
EXIT_CAUGHT, EXIT_NOT_CAUGHT = 5, 4
FAILED, FINDINGS = [], []
# what the comparison and the scorer found, read by the controls: the conversations whose decision differs, the sums
# outside the tolerance, and the scorer's refusal
SEEN = {"flips": None, "outside": None, "refused": None}


def report(name, ok, detail=""):
    print("%s  %s%s" % ("PASS" if ok else "FAIL", name, ("  -- " + str(detail)) if detail else ""), flush=True)
    if not ok:
        FAILED.append(name)
        FINDINGS.append("%s  -- %s" % (name, detail))
    return ok


def note(text):
    print("      " + text, flush=True)


def sha256_file(path):
    h = hashlib.sha256()
    with io.open(path, "rb") as f:
        for b in iter(lambda: f.read(8 << 20), b""):
            h.update(b)
    return h.hexdigest()


def git_blob_sha1(path):
    h = hashlib.sha1(b"blob %d\0" % os.path.getsize(path))
    with io.open(path, "rb") as f:
        for b in iter(lambda: f.read(8 << 20), b""):
            h.update(b)
    return h.hexdigest()


def bare(v):
    """A hash as bare lower-case hex: the record writes sha256:<hex>, score.json bare hex."""
    v = str(v).lower()
    return v[7:] if v.startswith("sha256:") else v


def no_constant(c):
    raise ValueError("%s is not JSON" % c)


def load_json(path):
    with io.open(path, "rb") as f:
        raw = f.read()
    return raw, json.loads(raw.decode("utf-8"), parse_constant=no_constant)


def canonical(obj):
    """score.json's own form: what score_llm.py writes."""
    return (json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False)
            + "\n").encode("ascii")


def decision_digest(conversations):
    return hashlib.sha256("".join("%s %d\n" % (c["name"], c["decision"]) for c in conversations)
                          .encode("ascii")).hexdigest()


def same_typed(a, b, depth=0):
    """Equal, and every number the same type: in Python 1 == 1.0 == True, which a declaration must not allow. Nothing
    nested deeper than score.json ever is (a few levels) compares equal."""
    if type(a) is not type(b) or depth > 16:
        return False
    if isinstance(a, dict):
        return set(a) == set(b) and all(same_typed(a[k], b[k], depth + 1) for k in a)
    if isinstance(a, list):
        return len(a) == len(b) and all(same_typed(x, y, depth + 1) for x, y in zip(a, b))
    return a == b


def finite_hex(v):
    """A sum as score.json writes it (float.hex), as a finite number; None for anything else, NaN and infinity too."""
    if not isinstance(v, str):
        return None
    try:
        x = float.fromhex(v)
    except (ValueError, OverflowError):
        return None
    return x if math.isfinite(x) else None


def within(x, y):
    """Two finite sums agree within REL_TOL: relatively, or absolutely for sums smaller than one. NaN never does."""
    return abs(x - y) <= REL_TOL * max(abs(x), abs(y), 1.0)


def within_rel(x, y):
    """Two losses (averages a token, often below one) agree within REL_TOL, relatively only. NaN never does."""
    return abs(x - y) <= REL_TOL * max(abs(x), abs(y))


def own_margin(convs):
    """score_llm.py's smallest_margin, from the file's own sums: the smallest non-zero |trained - base|, or None."""
    nz = [m for m in (abs(float.fromhex(c["trained"]) - float.fromhex(c["base"])) for c in convs) if m > 0]
    return min(nz).hex() if nz else None


def decided(c):
    """score_llm.py's decision, from a conversation's own two sums."""
    b, t = float.fromhex(c["base"]), float.fromhex(c["trained"])
    return 1 if t > b else -1 if t < b else 0


def check_model(folder, files=MODEL_FILES):
    """Every file of the pinned commit, by size and then by hash. Returns a fresh folder holding links to exactly
    those files and nothing else, so a stray file in yours (an unsharded model.safetensors, say, which Transformers
    would load first) cannot change what the scorer loads; or None."""
    ok = True
    for name, size, want in files:
        p = os.path.join(folder, name)
        if not os.path.isfile(p):
            ok = report("model: %s is present" % name, False, "not in %s" % folder)
            continue
        if os.path.getsize(p) != size:
            ok = report("model: %s" % name, False, "%d bytes, not %d" % (os.path.getsize(p), size))
            continue
        got = sha256_file(p) if len(want) == 64 else git_blob_sha1(p)
        if got != want:
            ok = report("model: %s" % name, False, "hash %s, not %s" % (got[:16], want[:16]))
    if not ok:
        return None
    clean = tempfile.mkdtemp(prefix="nexido-model-")
    for name, size, want in files:
        source, target = os.path.abspath(os.path.join(folder, name)), os.path.join(clean, name)
        try:
            os.symlink(source, target)
        except OSError:
            try:
                os.link(source, target)
            except OSError as x:
                report("model: a folder of links to the checked files", False, "%s: %s" % (type(x).__name__, x))
                return None
    report("model: the %d files of %s at %s, each by its hash" % (len(files), REPO, REV[:12]), True)
    return clean


def check_scorer(here=HERE):
    """The four scorer files, copied into a fresh folder and checked THERE, so what runs is what was checked: the
    scorer runs from that folder, where no compiled copy and no other module can stand in for them. The folder, or
    None."""
    fresh = tempfile.mkdtemp(prefix="nexido-scorer-")
    ok = True
    for name, want in sorted(SCORER.items()):
        p, q = os.path.join(here, name), os.path.join(fresh, name)
        got = "missing"
        if os.path.isfile(p):
            shutil.copyfile(p, q)
            got = sha256_file(q)
        ok = report("scorer: %s is the image's own" % name, got == want, "sha256 %s" % got[:16]) and ok
    return fresh if ok and sorted(os.listdir(fresh)) == sorted(SCORER) else None


def check_stack():
    """Information only: a different version should still give the same decisions, not the same bits."""
    diff = []
    from importlib import metadata
    for mod, want in STACK:
        try:
            got = metadata.version(mod)
        except Exception:
            got = "not installed"
        if got.split("+")[0] != want:
            diff.append("%s %s (ours %s)" % (mod, got, want))
    py = "%d.%d.%d" % sys.version_info[:3]
    if sys.version_info[:2] != (3, 10):
        diff.append("Python %s (ours 3.10)" % py)
    note("this machine: Python %s; %s" % (py, "; ".join(diff) if diff else "the image's versions of %s" % ", ".join(
        m for m, v in STACK)))
    if diff:
        note("with other versions, expect the same decisions within the tolerance, not the same bits")


def well_formed(score):
    """The shape every later step relies on: each conversation an ASCII name, a decision and two finite sums, and
    the shard. A malformed file is refused here, with a sentence, never with a traceback later."""
    convs, shard = score.get("conversations"), score.get("shard")
    if not (isinstance(convs, list) and convs and isinstance(shard, list) and len(shard) == 2
            and all(type(x) is int for x in shard) and 0 <= shard[0] < shard[1]):
        return False
    for c in convs:
        if not (isinstance(c, dict) and isinstance(c.get("name"), str) and c["name"].isascii() and " " not in c["name"]
                and "\n" not in c["name"] and type(c.get("decision")) is int and c["decision"] in (-1, 0, 1)
                and finite_hex(c.get("base")) is not None and finite_hex(c.get("trained")) is not None):
            return False
    sm = score.get("summary")
    return isinstance(sm, dict) and all(finite_hex(sm.get(k)) is not None for k in ("loss_base", "loss_trained")) and (
        sm.get("smallest_margin") is None or finite_hex(sm.get("smallest_margin")) is not None)


def consistent(score):
    """The file agrees with itself, as score_llm.py writes it: each decision the sign of its own two sums, and the
    smallest margin the smallest of its own. A margin is the difference of two large sums, so it is checked here,
    exactly, against the file's own sums, and never against another chip's re-score."""
    convs = score["conversations"]
    return all(c["decision"] == decided(c) for c in convs) and score["summary"].get("smallest_margin") == own_margin(
        convs)


def pre_checks(a, rec, score_raw, score):
    """Everything that can be checked without a model. Returns the job's params, or None if the record cannot
    even describe one."""
    report("record: a %s" % RECORD_VERSION, rec.get("record_version") == RECORD_VERSION, rec.get("record_version"))
    job = rec.get("job") if isinstance(rec.get("job"), dict) else {}
    td = rec.get("training_data") if isinstance(rec.get("training_data"), dict) else {}
    out = rec.get("output") if isinstance(rec.get("output"), dict) else {}
    fields = (rec.get("receipt") or {}).get("fields") if isinstance(rec.get("receipt"), dict) else None
    receipt, split, part = td.get("count_receipt"), td.get("split"), td.get("part")
    if not report("record: a scoring stage of an LLM fine-tune (job.type train-llm, part val or test)",
                  job.get("type") == "train-llm" and part in ("val", "test"), "type %s, part %s" % (
                      job.get("type"), part)):
        return None
    image = str(job.get("image_digest"))
    if not report("record: names the scoring image this verifier is for, score-llm v1 (%s)" % SCORE_IMAGE[:19],
                  image == SCORE_IMAGE or image.endswith("@" + SCORE_IMAGE), "the record names %s" % image[:80]):
        note("this verifier re-scores score-llm v1's records only; a record of another scoring image needs that "
             "image's own verifier")
        return None
    if not report("record: the counter's receipt and the split's declaration, each exactly its fields",
                  isinstance(receipt, dict) and set(receipt) == set(RECEIPT_KEYS) and isinstance(split, dict)
                  and set(split) == set(SPLIT_KEYS) and type(split.get("seed")) is int):
        return None
    note("record: job %s, the %s part" % (str(job.get("id"))[:8], part))
    # 1. the score.json you kept is the one the node signed for
    signed = isinstance(fields, dict) and bare(fields.get("output_hash")) == bare(out.get("output_hash"))
    report("record: output.output_hash is the output_hash its signed receipt carries", signed)
    report("score.json is the record's output (its sha256 is that output_hash)",
           hashlib.sha256(score_raw).hexdigest() == bare(out.get("output_hash")),
           "sha256 %s" % hashlib.sha256(score_raw).hexdigest()[:16])
    # 2. what it says it scored is what the record says
    try:
        form = score_raw == canonical(score)
    except (ValueError, RecursionError):
        form = False
    report("score.json is in the scorer's own form (canonical JSON, one line)", form)
    inp = score.get("input") if isinstance(score.get("input"), dict) else {}
    ad = score.get("adapter") if isinstance(score.get("adapter"), dict) else {}
    report("score.json: kind %s, plan %d, the model %s at %s" % (SCORE_KIND, PLAN_VERSION, REPO, REV[:12]),
           score.get("kind") == SCORE_KIND and same_typed(score.get("plan_version"), PLAN_VERSION) and score.get(
               "model") == {"repo": REPO, "revision": REV})
    report("score.json: the part and split the record names", score.get("part") == part and same_typed(
        score.get("split"), split))
    report("score.json: the file the counter counted (its sha256, bytes, rows, tokens and row hash)",
           all(same_typed(inp.get(k), receipt.get(k)) for k in ("file_sha256", "bytes", "rows", "tokens",
                                                                  "row_tokens_sha256")))
    report("score.json: the adapter the record says this stage graded (training_data.graded_adapter_hash)",
           isinstance(ad.get("sha256"), str) and ad.get("sha256") == bare(td.get("graded_adapter_hash")),
           "%s" % str(ad.get("sha256"))[:16])
    if not report("score.json: its conversations (ASCII names, decisions, finite sums), its summary and its shard",
                  well_formed(score)):
        return None
    convs, shard = score["conversations"], score["shard"]
    report("score.json: each decision is the sign of its own two sums, and its smallest margin the smallest of its "
           "own", consistent(score))
    report("score.json: decision_sha256 is the digest of its own decisions",
           decision_digest(convs) == score.get("decision_sha256"), str(score.get("decision_sha256"))[:16])
    sm = score["summary"]
    tally = [sum(1 for c in convs if c["decision"] == d) for d in (1, -1, 0)]
    report("score.json: its summary counts its own decisions", [sm.get("m"), sm.get("better"), sm.get("worse"),
                                                                  sm.get("tied")] == [len(convs)] + tally,
           "better %d, worse %d, tied %d of %d" % (tally[0], tally[1], tally[2], len(convs)))
    # 3. your files
    report("adapter: the file score.json names (sha256)", sha256_file(a.adapter) == ad.get("sha256"))
    report("your file: the file the counter counted (sha256 == count_receipt.file_sha256)",
           sha256_file(a.input) == receipt.get("file_sha256") and os.path.getsize(a.input) == receipt.get("bytes"))
    return {"plan_version": PLAN_VERSION, "part": part, "split": split, "receipt": receipt,
            "shard": {"index": shard[0], "count": shard[1]}}


def broken_adapter(src, dst, params, scorer):
    """BREAK=adapter: the same adapter with every lora_B zeroed, written by the scorer's own writer (checked above
    to be the image's own), so it is a valid file the scorer accepts; the trained model is then the starting model,
    exactly."""
    sys.path.insert(0, scorer)
    import lora_file
    import torch
    from safetensors.torch import load_file
    meta = json.loads(lora_file.read(src)[1]["nexido"])
    t = load_file(src)
    for k in t:
        if ".lora_B." in k:
            t[k] = torch.zeros_like(t[k])
    lo = meta["lora"]
    lora_file.write(dst, t, lo["r"], lo["lora_alpha"], lo["target_modules"], params["plan_version"],
                    params["split"], params["receipt"])


def run_scorer(a, params, model_dir, work, data, adapter, scorer):
    jf, out = os.path.join(work, "job.json"), os.path.join(work, "out", "score.json")
    if os.path.exists(out):
        os.remove(out)
    with io.open(jf, "w", encoding="utf-8") as f:
        json.dump({"params_json": params}, f)
    env = dict(os.environ, NEXIDO_IN=jf, NEXIDO_OUT=out, NEXIDO_SCORER_MODEL=model_dir,
               NEXIDO_SCORER_DEVICE=a.device, PYTHONPYCACHEPREFIX=tempfile.mkdtemp(prefix="nexido-nocache-"),
               **SCORER_ENV)
    if a.device.startswith("cuda"):
        env.update(GPU_ENV)
    print("\nRUN   the scorer, as a node runs it, on %s (minutes on a GPU, longer on a CPU) ..." % a.device, flush=True)
    # its progress shown as it comes; the paths absolute, so none can be read as an option
    p = subprocess.Popen([sys.executable, "-B", os.path.join(scorer, "score_llm.py"), os.path.abspath(data),
                          os.path.abspath(adapter)], env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
                         stderr=subprocess.PIPE)
    refused, tail = [], []
    for raw in p.stderr:
        line = raw.decode("utf-8", "replace").rstrip("\n")
        if line.startswith("score_llm: "):
            note(line)
        elif line.startswith("NEXIDO_REFUSED "):
            refused.append(line.split(" ", 1)[1])
        tail = (tail + [line])[-40:]
    rc = p.wait()
    SEEN["refused"] = refused[0].split(" ")[0] if refused else None
    if rc != 0 or not os.path.isfile(out):
        report("the scorer ran to the end", False, "refused: %s" % refused[0] if refused else "exit %d" % rc)
        if not refused or SEEN["refused"] in ("crashed", "out_of_memory"):
            sys.stderr.write("\n".join(tail) + "\n")
        return None
    report("the scorer ran to the end", True)
    with io.open(out, "rb") as f:
        return f.read()


def compare(kept_raw, kept, new_raw):
    """The verdict: bit for bit, or everything the same but the sums' last digits, or not reproduced."""
    new = json.loads(new_raw.decode("ascii"))
    if new_raw == kept_raw:
        report("the re-score is your score.json, byte for byte", True, "sha256 %s" % hashlib.sha256(
            new_raw).hexdigest()[:16])
        return "bit"
    note("the re-score is not your score.json byte for byte: comparing it field by field")
    report("the re-score scored the same split, file, adapter, model and method", all(
        same_typed(new.get(k), kept.get(k)) for k in set(new) | set(kept)
        if k not in ("conversations", "summary", "decision_sha256")))
    kc, nc = kept.get("conversations", []), new.get("conversations", [])
    sums = ("base", "trained")
    if not report("the re-score scored the same conversations, in the same order, with the same tokens",
                  len(kc) == len(nc) and all(set(c) == set(d) and all(same_typed(c[k], d[k]) for k in c
                                                                      if k not in sums + ("decision",))
                                             for c, d in zip(kc, nc)), "%d and %d" % (len(kc), len(nc))):
        return None
    flips = [c["name"] for c, d in zip(kc, nc) if c["decision"] != d["decision"]]
    SEEN["flips"] = flips
    report("the same decision on every conversation", not flips, "%d differ: %s" % (len(flips), ", ".join(
        f[:16] for f in flips[:5])) if flips else "%d of %d" % (len(kc), len(kc)))
    report("the same decision_sha256", new.get("decision_sha256") == kept.get("decision_sha256"),
           str(new.get("decision_sha256"))[:16])
    ks, ns = kept.get("summary", {}), new.get("summary", {})
    floats = ("loss_base", "loss_trained", "smallest_margin")
    report("the same summary counts", set(ks) == set(ns) and all(same_typed(ks[k], ns[k]) for k in ks
                                                                 if k not in floats))
    # THE RE-SCORE'S OWN MARGIN, from its own sums, exactly: a margin is a difference of two large sums, so another
    # chip's noise moves it by far more than its sums; it is never compared with yours by a tolerance
    report("the re-score's smallest margin is the smallest of its own", well_formed(new) and consistent(new))
    pairs = [("%s %s" % (c["name"], k), c[k], d[k], within) for c, d in zip(kc, nc) for k in sums]
    pairs += [("summary " + k, ks.get(k), ns.get(k), within_rel) for k in ("loss_base", "loss_trained")]
    worst_rel, worst_abs, where, bad = 0.0, 0.0, None, []
    for label, xs, ys, agree in pairs:
        x, y = finite_hex(xs), finite_hex(ys)
        if x is None or y is None:
            bad.append(label)
            continue
        if not agree(x, y):
            bad.append(label)
        r = abs(x - y) / max(abs(x), abs(y), 1.0)
        worst_abs = max(worst_abs, abs(x - y))
        if r > worst_rel:
            worst_rel, where = r, "%s: ours %r, yours %r" % (label[:24], x, y)
    SEEN["outside"] = bad
    report("every sum within %g of ours (relatively, or absolutely below one; the losses relatively)" % REL_TOL,
           not bad,
           ("%d outside, the first %s; " % (len(bad), bad[0][:24]) if bad else "") + (
               "largest %.3g (%s)" % (worst_rel, where) if where else "every sum identical"))
    margin = finite_hex(ks.get("smallest_margin"))
    if margin is not None:
        note("the closest decision in your score.json was by %.6g; the largest difference here is %.3g" % (
            margin, worst_abs))
    return None if FAILED else "tol"


def apply_break(BREAK, score, params, a, work, scorer):
    """One control, applied AFTER every check above has passed, so none of them can catch it; the check named for it
    must (others may fail beside it). Returns (the kept copy to compare with, its bytes, the job's params, the file and
    the adapter the scorer is given, and what must be seen for the control to count as caught)."""
    target = None
    data, adapter = a.input, a.adapter
    if BREAK == "flip":
        c0 = score["conversations"][0]
        c0["decision"] = -1 if c0["decision"] == 1 else 1
        score["decision_sha256"] = decision_digest(score["conversations"])
        target = c0["name"]
        note("BREAK=flip: conversation %s's decision changed to %d in your copy, its digest recomputed" % (
            c0["name"][:16], c0["decision"]))
    elif BREAK == "sum":
        c0 = score["conversations"][0]
        x = float.fromhex(c0["base"])
        c0["base"] = (x - max(abs(x), 1.0) * 1e-3).hex()
        target = "%s base" % c0["name"]
        note("BREAK=sum: conversation %s's starting sum moved by a thousandth (at least 0.001) in your copy" % (
            c0["name"][:16]))
    elif BREAK == "seed":
        params = dict(params, split=dict(params["split"], seed=params["split"]["seed"] + 1))
        target = "split_refused"
        note("BREAK=seed: the job's split seed changed to %d" % params["split"]["seed"])
    elif BREAK == "file":
        data = os.path.join(work, "input-short.jsonl")
        with io.open(a.input, "rb") as f:
            lines = f.read().rstrip(b"\n").split(b"\n")
        with io.open(data, "wb") as f:
            f.write(b"\n".join(lines[:-1]) + b"\n")
        target = "input_not_the_counted_file"
        note("BREAK=file: your file without its last row, %s" % data)
    elif BREAK == "adapter":
        adapter = os.path.join(work, "adapter-zeroed.safetensors")
        try:
            broken_adapter(a.adapter, adapter, params, scorer)
        except Exception as x:
            report("BREAK=adapter: the adapter rewritten with every lora_B zeroed", False, "%s" % type(x).__name__)
            return None
        target = "any"
        note("BREAK=adapter: the adapter with every lora_B zeroed, %s" % adapter)
    return score, canonical(score), params, data, adapter, target


def control_caught(BREAK, target):
    """Whether the control was caught by the check named for it, for the change it made."""
    if BREAK == "flip":
        return SEEN["flips"] == [target]
    if BREAK == "sum":
        return SEEN["outside"] is not None and target in SEEN["outside"]
    if BREAK == "adapter":
        return bool(SEEN["flips"])
    return SEEN["refused"] == target


def main(argv=None):
    ap = argparse.ArgumentParser(description="Re-score a Nexido LLM fine-tune's scoring stage on your machine.")
    ap.add_argument("--record", required=True, help="the scoring stage's verification record (JSON)")
    ap.add_argument("--score", required=True, help="the scoring stage's score.json, as you downloaded it")
    ap.add_argument("--adapter", required=True, help="the training stage's adapter (.safetensors)")
    ap.add_argument("--input", required=True, help="your training file, as you uploaded it")
    ap.add_argument("--model", required=True, help="the folder holding %s at %s" % (REPO, REV))
    ap.add_argument("--device", default="cuda:0", help="cuda:0 (default) or cpu")
    ap.add_argument("--work", default=None, help="a folder for the job and the re-score (default: a new temporary one)")
    ap.add_argument("--unpinned-model", action="store_true",
                    help="for testing this script only: use --model without checking it; exits 3, never 0")
    a = ap.parse_args(argv)
    BREAK = os.environ.get("BREAK", "")
    if BREAK and BREAK not in BREAKS:
        raise SystemExit("BREAK must be one of: %s" % ", ".join(sorted(BREAKS)))
    del FAILED[:], FINDINGS[:]
    SEEN.update(flips=None, outside=None, refused=None)
    print("NEXIDO -- re-scoring an LLM fine-tune's scoring stage%s" % ("  [BREAK=%s]" % BREAK if BREAK else ""))
    for k in ("record", "score", "adapter", "input"):
        if not os.path.isfile(getattr(a, k)):
            raise SystemExit("--%s: no such file: %s" % (k, getattr(a, k)))
    if not a.unpinned_model and not os.path.isdir(a.model):
        raise SystemExit("--model: no such folder: %s" % a.model)
    try:
        _, rec = load_json(a.record)
        score_raw, score = load_json(a.score)
    except (OSError, ValueError, RecursionError) as x:
        raise SystemExit("the record and score.json must be the JSON files from the job page: %s" % x)
    if not (isinstance(rec, dict) and isinstance(score, dict)):
        raise SystemExit("the record and score.json must each be a JSON object")
    work = a.work or tempfile.mkdtemp(prefix="nexido-reproduce-")
    os.makedirs(work, exist_ok=True)
    print()
    params = pre_checks(a, rec, score_raw, score)
    scorer = check_scorer()
    scorer_ok = scorer is not None
    check_stack()
    model_dir = None
    if a.unpinned_model:
        model_dir = a.model
        note("--unpinned-model: the model folder is NOT checked; this run is a test of the script, not a verification")
    elif scorer_ok and params is not None and not FAILED:
        model_dir = check_model(a.model)
    verdict, target = None, None
    if params is None or FAILED or model_dir is None:
        print("\nNOT SCORED: %d check(s) failed before scoring; nothing was computed" % len(FAILED))
    else:
        data, adapter = a.input, a.adapter
        broken = apply_break(BREAK, score, params, a, work, scorer) if BREAK else None
        if BREAK and broken is not None:
            score, score_raw, params, data, adapter, target = broken
        if not BREAK or broken is not None:
            new_raw = run_scorer(a, params, model_dir, work, data, adapter, scorer)
            if new_raw is not None:
                verdict = compare(score_raw, score, new_raw)
    if verdict is not None and FAILED:
        verdict = None
    test = "TEST ONLY, THE MODEL NOT CHECKED: " if a.unpinned_model else ""
    print()
    if verdict == "bit":
        print("%sREPRODUCED BIT FOR BIT: your machine wrote the same score.json, byte for byte" % test)
    elif verdict == "tol":
        print("%sREPRODUCED: the same decision on every conversation (decision_sha256 %s), every sum within %g" % (
            test, score.get("decision_sha256", "")[:16], REL_TOL))
    else:
        print("%sNOT REPRODUCED: %d check(s) failed: %s" % (test, len(FAILED), "; ".join(FAILED[:6])))
    print("Reminder: run verify-record.mjs on the same record to check the node's signature over output_hash.")
    if BREAK:
        what = {"flip": "conversation %s's decision differs" % str(target)[:16],
                "sum": "the moved sum is outside the tolerance", "adapter": "decisions differ",
                "seed": "the scorer refused the split", "file": "the scorer refused the file"}[BREAK]
        if verdict is None and target is not None and control_caught(BREAK, target):
            print("CONTROL CAUGHT: BREAK=%s, as it must be: %s" % (BREAK, what))
            return EXIT_CAUGHT
        print("CONTROL NOT CAUGHT: BREAK=%s was not caught for its change (%s): do not trust a clean run" % (
            BREAK, what))
        return EXIT_NOT_CAUGHT
    if verdict in ("bit", "tol"):
        return 3 if a.unpinned_model else 0
    return 1


if __name__ == "__main__":
    sys.exit(main())
