# THE 4B SCORER (ADR-0067), the entrypoint of the score-llm image. One job scores one part of the server's split
# ("val" or "test") of one customer's training file, without and with one trained adapter, on one card:
#   for each held-out conversation, the exact sum (math.fsum) of the fp32 log-probabilities of its scored tokens,
#   from the starting model and from the trained model, in this process; the decision is which is higher.
# Findings 139 and 140: plain ChatML; each assistant turn's content and its <|im_end|>; the adapter attached
# unmerged before anything goes to the card and checked against its file; the decoder's weights stored bf16 and
# computed fp32, every layer on the card, the same bits as the fp32 model (block 119).
#
# THE JOB (ADR-0067 D3). Everything comes from /in/job.json, which the server wrote; params_json is
# customer-adjacent by the time it reaches a node, so every field is checked against the bytes it describes:
#   plan_version  1, or this image refuses
#   part          "val" or "test"
#   split         the server's declaration (Track A's split rule over the conversations' names), re-derived here
#   receipt       the token counter's receipt for the file, re-counted here
#   shard         optional {"index", "count"}: one contiguous slice of the part's conversations, in name order
#   input, output the server's own instructions to the agent; not read here
# The arguments are two files, in any order: the customer's file (/in/input), recognised by the receipt's
# file_sha256, and the trainer's adapter (/in/up0). The result is one JSON file, the same bytes on every run of
# the same inputs on the same chip; nothing timed and nothing about the machine goes in it. Standard output carries
# only the NEXIDO_RESULT line the agent reads: file descriptor 1 itself is pointed at standard error, so not even a
# native library's print can reach the agent's 1 MiB line scanner. Everything else goes to standard error, without
# customer text.
import faulthandler, hashlib, io, json, math, os, shutil, sys, tempfile, time
T0 = time.monotonic()
faulthandler.dump_traceback_later(12 * 3600, exit=True)
OUT = os.fdopen(os.dup(1), "w")
os.dup2(2, 1)
sys.stdout = sys.stderr
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import llm_rows as L
import lora_file as A
import split_rule as SR
JOB_FILE = os.environ.get("NEXIDO_IN", "/in/job.json")
OUT_PATH = os.environ.get("NEXIDO_OUT", "/out/score.json")
MODEL = os.environ.get("NEXIDO_SCORER_MODEL", "/models/qwen3-4b-instruct-2507")
DEVICE = os.environ.get("NEXIDO_SCORER_DEVICE", "cuda:0")
# layers, hidden width and vocabulary of the model this image holds; a test's stand-in model declares its own
EXPECT = tuple(int(x) for x in os.environ.get("NEXIDO_SCORER_EXPECT", "36,2560,151936").split(","))
PLAN_VERSION, KIND, CHUNK, SEED = 1, "nexido-score-llm-v1", 256, 20261009
PARAMS = ("plan_version", "part", "split", "receipt", "shard", "input", "output")
RECEIPT = ("version", "job_id", "row", "tokenizer_sha256", "render", "limits", "counter_revision", "file_sha256",
           "bytes", "rows", "tokens", "row_tokens_sha256")
HEX64 = set("0123456789abcdef")
def say(*a):
    sys.stderr.write("score_llm: " + " ".join(str(x) for x in a) + "\n")
    sys.stderr.flush()
def die(reason, detail=""):
    say("REFUSED", reason, "--", detail)
    sys.stderr.write("NEXIDO_REFUSED %s\n" % reason)
    sys.stderr.flush()
    os._exit(2)
def no_constant(c):
    raise ValueError("%s is not JSON" % c)
def is_int(v):
    return type(v) is int
def hex64(v):
    return isinstance(v, str) and len(v) == 64 and set(v) <= HEX64
def sha256_file(p):
    h = hashlib.sha256()
    with io.open(p, "rb") as f:
        for b in iter(lambda: f.read(8 << 20), b""):
            h.update(b)
    return h.hexdigest()
def job():
    try:
        with io.open(JOB_FILE, encoding="utf-8") as f:
            j = json.load(f, parse_constant=no_constant)
    except (OSError, ValueError, RecursionError) as x:
        die("bad_job_file", "%s: %s" % (JOB_FILE, type(x).__name__))
    p = j.get("params_json") if isinstance(j, dict) else None
    if not isinstance(p, dict):
        die("bad_params", "params_json is not an object")
    extra = sorted(set(p) - set(PARAMS))
    if extra:
        die("unknown_param", "%d: %s" % (len(extra), ", ".join(x if x.isascii() and x.isidentifier() and len(
            x) <= 40 else "?" for x in extra)[:120]))
    if not is_int(p.get("plan_version")) or p["plan_version"] != PLAN_VERSION:
        die("plan_version", "params_json.plan_version is %s; this image implements %d" % (
            repr(p.get("plan_version"))[:40], PLAN_VERSION))
    if p.get("part") not in ("val", "test"):
        die("bad_part", "part must be val or test")
    sp = p.get("split")
    # every number an integer, never true or 1.0: the split rule itself would read a rule_version of true as 1
    if not (isinstance(sp, dict) and set(sp) == set(A.SPLIT_KEYS) and all(is_int(sp[k]) for k in A.SPLIT_KEYS[:7])
            and all(sp[k] >= 0 for k in A.SPLIT_KEYS[4:7]) and all(hex64(sp[k]) for k in A.SPLIT_KEYS[7:])):
        die("bad_split", "params_json.split must be the server's declaration, exactly")
    r = p.get("receipt")
    if not isinstance(r, dict) or set(r) != set(RECEIPT):
        die("bad_receipt", "the receipt must hold exactly the counter's twelve fields")
    lim = dict((k, L.LIMITS[k]) for k in L.LIMITS)
    if not (r["version"] == L.COUNT_VERSION and r["row"] == L.ROW and r["tokenizer_sha256"] == L.TOKENIZER_SHA256
            and r["render"] == L.RENDER_RULE and r["limits"] == lim and all(type(v) is int for v in r[
                "limits"].values()) and hex64(r["file_sha256"]) and hex64(r["row_tokens_sha256"]) and all(
                is_int(r[k]) and r[k] >= 0 for k in ("bytes", "rows", "tokens"))):
        die("bad_receipt", "a receipt for another row, rule, tokenizer or limit, or a malformed one")
    s = p.get("shard", {"index": 0, "count": 1})
    if not (isinstance(s, dict) and set(s) == {"index", "count"} and is_int(s["index"]) and is_int(s["count"])
            and 1 <= s["count"] <= 1000 and 0 <= s["index"] < s["count"]):
        die("bad_shard", "shard must be {index, count} with 0 <= index < count <= 1000")
    return p, s
def main():
    P, SH = job()
    R = P["receipt"]
    args = sys.argv[1:]
    if len(args) != 2 or any(a.startswith("-") or not os.path.isfile(a) for a in args):
        die("bad_args", "exactly two files: the customer's file and the adapter")
    sums = [sha256_file(a) for a in args]
    if sums.count(R["file_sha256"]) != 1:
        die("input_not_the_counted_file", "exactly one argument must be the file the receipt counted")
    data_path, ad_path = (args[0], args[1]) if sums[0] == R["file_sha256"] else (args[1], args[0])
    if os.path.getsize(data_path) != R["bytes"]:
        die("receipt_mismatch", "bytes")
    # ---- the customer's file, read exactly as the counter read it, and every figure on the receipt checked
    tok, control = L.load_tokenizer(os.path.join(MODEL, "tokenizer.json"))
    with io.open(data_path, "rb") as f:
        data = f.read(L.LIMITS["max_bytes"] + 1)
    t = time.monotonic()
    try:
        got = L.read_file(data, tok, control, keep=True)
    except L.Refusal as x:
        die("input_refused", "%s at row %s, line %s" % (x.reason, x.row, x.line))
    del data
    for k in ("file_sha256", "bytes", "rows", "tokens", "row_tokens_sha256"):
        if got[k] != R[k]:
            die("receipt_mismatch", k)
    say("read and recounted %d rows, %d tokens, in %.1f s; every figure equals the receipt" % (
        got["rows"], got["tokens"], time.monotonic() - t))
    # ---- the split, re-derived over the conversations' names and checked against the declaration
    convs = {}
    for msgs, text, row, line in got["conversations"]:
        convs.setdefault(L.name_of(text), (msgs, text))
    try:
        parts = SR.select(sorted(convs), P["split"])
    except SR.SplitRefusal as x:
        die("split_refused", "%s: %s" % (x.reason, x.detail[:150]))
    if [len(parts[k]) for k in ("train", "val", "test")] != [P["split"][k] for k in ("n_train", "n_val", "n_test")]:
        die("split_counts", "the declaration's counts are not the counts its digests describe")
    names = parts[P["part"]]
    n, i, c = len(names), SH["index"], SH["count"]
    names = names[n * i // c:n * (i + 1) // c]
    if not names:
        die("empty_part", "the %s part, shard %d of %d, holds no conversation" % (P["part"], i, c))
    say("split re-derived: %d conversations (%d rows repeat one); %s holds %d; this shard scores %d" % (
        len(convs), got["rows"] - len(convs), P["part"], n, len(names)))
    # ---- the model, built in memory in bf16
    import torch
    from transformers import AutoModelForCausalLM
    from peft import LoraConfig, PeftModel
    from safetensors.torch import load_file
    from torch.nn.utils import parametrize
    from torch.nn.attention import sdpa_kernel, SDPBackend
    torch.use_deterministic_algorithms(True)
    torch.backends.cudnn.benchmark, torch.backends.cudnn.deterministic = False, True
    torch.backends.cuda.matmul.allow_tf32, torch.backends.cudnn.allow_tf32 = False, False
    torch.manual_seed(SEED)
    gpu = DEVICE.startswith("cuda")
    if gpu:
        free = torch.cuda.mem_get_info()[0] >> 20
        if free < 11 * 1024:
            die("card_too_small", "%d MiB free; the 4B needs about 10 GiB at 2,048 tokens" % free)
    t = time.monotonic()
    model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16, local_files_only=True,
                                                 attn_implementation="sdpa").eval()
    cfg = model.config
    if (cfg.num_hidden_layers, cfg.hidden_size, cfg.vocab_size) != EXPECT or cfg.tie_word_embeddings is not True:
        die("model_mismatch", "not the model this image was built for")
    lin = dict((k, m) for k, m in model.named_modules() if isinstance(m, torch.nn.Linear))
    # ---- the adapter: every byte and every field checked, then attached unmerged, before anything goes to the card
    try:
        ad_sha, lo, ad_info = A.check(ad_path, PLAN_VERSION, P["split"], R, lin, cfg.num_hidden_layers)
    except A.AdapterRefusal as x:
        die(x.reason, x.detail[:200])
    ft = load_file(ad_path)
    if not all(bool(torch.isfinite(v).all()) for v in ft.values()):
        die("adapter_not_finite", "a tensor holds NaN or infinity")
    tmp = tempfile.mkdtemp()
    LoraConfig(r=lo["r"], lora_alpha=lo["lora_alpha"], target_modules=lo["target_modules"], lora_dropout=0.0,
               bias="none", task_type="CAUSAL_LM").save_pretrained(tmp)
    shutil.copyfile(ad_path, os.path.join(tmp, "adapter_model.safetensors"))
    pm = PeftModel.from_pretrained(model, tmp, is_trainable=False).eval()
    inner = pm.base_model.model
    loaded = dict((k, v) for k, v in pm.named_parameters() if ".lora_" in k)
    if len(loaded) != len(ft) or not all(loaded[k.replace(".weight", ".default.weight")].dtype == torch.float32 and
                                         torch.equal(loaded[k.replace(".weight", ".default.weight")], v.float())
                                         for k, v in ft.items()):
        die("adapter_not_loaded_exactly", "a loaded tensor differs from the file, or is not fp32")
    # ---- bf16 stored, fp32 computed (Finding 140): every decoder matrix upcast when used; the rest fp32
    class Up(torch.nn.Module):
        def forward(self, w):
            return w.float()
    for k, m in list(inner.named_modules()):
        if isinstance(m, torch.nn.Linear) and ".layers." in k and "lora_" not in k:
            parametrize.register_parametrization(m, "weight", Up(), unsafe=True)
    inner.to(0 if gpu else torch.device("cpu"))
    inner.get_input_embeddings().to(torch.float32)
    for m in inner.modules():
        if type(m).__name__.endswith("RMSNorm"):
            m.to(torch.float32)
    stored = {}
    for p_ in inner.parameters():
        stored[str(p_.dtype)[6:]] = stored.get(str(p_.dtype)[6:], 0) + 1
    want = {"bfloat16": 7 * cfg.num_hidden_layers, "float32": 2 + 4 * cfg.num_hidden_layers + 2 * len(
        lo["target_modules"]) * cfg.num_hidden_layers}
    if stored != want or inner.lm_head.weight is not inner.get_input_embeddings().weight:
        die("placement", "stored %s, expected %s" % (json.dumps(stored, sort_keys=True), json.dumps(want)))
    if gpu:
        torch.cuda.synchronize()
    say("model and adapter on %s in %.1f s: %s; adapter %s" % (DEVICE, time.monotonic() - t, json.dumps(
        stored, sort_keys=True), ad_sha[:16]))
    def score(ids, ans):
        x = torch.tensor([ids], device=DEVICE)
        with torch.no_grad(), sdpa_kernel(SDPBackend.MATH):
            h = inner.model(input_ids=x, use_cache=False).last_hidden_state[0]
            tg = torch.tensor([ids[q] for q in ans], device=h.device)
            ps = torch.tensor([q - 1 for q in ans], device=h.device)
            v = torch.cat([torch.log_softmax(inner.lm_head(h[ps[a:a + CHUNK]]).float(), dim=-1).gather(
                1, tg[a:a + CHUNK, None])[:, 0] for a in range(0, len(ans), CHUNK)]).cpu().tolist()
        return v
    # ---- the scoring, in name order
    t, rows_out, all_b, all_t, mark = time.monotonic(), [], [], [], time.monotonic()
    for j, name in enumerate(names):
        msgs, text = convs[name]
        e = tok.encode(text, add_special_tokens=False)
        ans, cut = L.scored(e.offsets, msgs)
        if not ans or len(e.ids) > L.LIMITS["max_tokens_per_row"]:
            die("unscorable", "conversation %d of this shard" % j)
        with pm.disable_adapter():
            vb = score(e.ids, ans)
        vt = score(e.ids, ans)
        b, tr = math.fsum(vb), math.fsum(vt)
        all_b.extend(vb)
        all_t.extend(vt)
        rows_out.append({"name": name[len(L.NAME_PREFIX):].decode("ascii"), "tokens": len(e.ids), "scored": len(ans),
                         "straddling": cut, "base": b.hex(), "trained": tr.hex(),
                         "decision": 1 if tr > b else -1 if tr < b else 0})
        if time.monotonic() - mark > 60:
            mark = time.monotonic()
            say("scored %d of %d" % (j + 1, len(names)))
    secs = time.monotonic() - t
    better = sum(1 for x in rows_out if x["decision"] == 1)
    worse = sum(1 for x in rows_out if x["decision"] == -1)
    margins = [abs(float.fromhex(x["trained"]) - float.fromhex(x["base"])) for x in rows_out]
    nz = [m for m in margins if m > 0]
    st = len(all_b)
    dec = "".join("%s %d\n" % (x["name"], x["decision"]) for x in rows_out)
    result = {
        "kind": KIND, "plan_version": PLAN_VERSION, "part": P["part"], "shard": [i, c],
        "split": dict((k, P["split"][k]) for k in A.SPLIT_KEYS),
        "input": {"file_sha256": R["file_sha256"], "bytes": R["bytes"], "rows": R["rows"], "tokens": R["tokens"],
                  "row_tokens_sha256": R["row_tokens_sha256"], "conversations": len(convs),
                  "repeated_rows": got["rows"] - len(convs), "part_conversations": n},
        "model": {"repo": A.REPO, "revision": A.REV},
        "adapter": dict(lo, sha256=ad_sha, **ad_info),
        "method": {"render": L.RENDER_RULE, "scored": "each assistant turn's content and its <|im_end|>, whole "
                   "tokens only", "weights": "bf16 stored, fp32 computed", "attention": "sdpa math",
                   "log_softmax_chunk": CHUNK, "sum": "math.fsum of fp32 log-probabilities"},
        "conversations": rows_out,
        "summary": {"m": len(rows_out), "better": better, "worse": worse, "tied": len(rows_out) - better - worse,
                    "scored_tokens": st, "loss_base": (-math.fsum(all_b) / st).hex(),
                    "loss_trained": (-math.fsum(all_t) / st).hex(), "smallest_margin": min(nz).hex() if nz else None},
        "decision_sha256": hashlib.sha256(dec.encode("ascii")).hexdigest(),
    }
    body = (json.dumps(result, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False)
            + "\n").encode("ascii")
    os.makedirs(os.path.dirname(OUT_PATH) or ".", exist_ok=True)
    with io.open(OUT_PATH + ".part", "wb") as f:
        f.write(body)
        f.flush()
        os.fsync(f.fileno())
    os.replace(OUT_PATH + ".part", OUT_PATH)
    say("%d conversations scored in %.1f s: better %d, worse %d, tied %d; %s, sha256 %s" % (
        len(rows_out), secs, better, worse, len(rows_out) - better - worse, OUT_PATH,
        hashlib.sha256(body).hexdigest()[:16]))
    OUT.write('NEXIDO_RESULT {"gpu_seconds": %.3f}\n' % (time.monotonic() - T0))
    OUT.flush()
if __name__ == "__main__":
    try:
        main()
    except SystemExit:
        raise
    except BaseException as x:
        import traceback
        traceback.print_exc()
        oom = "out of memory" in str(x).lower()
        die("out_of_memory" if oom else "crashed", type(x).__name__)
