# THE TOKEN COUNTER'S RULES, IN PYTHON, for the images that read a training file after the counter has priced it:
# the 4B scorer now, the trainer next. The counter is control-plane/src/verify/token-count.ts (sha16
# c7273a48b8b3e4be); this file must read every file exactly as it does. A row, a refusal (reason, row and line),
# a rendering, a token count and both hashes must come out the same. DELL_BLOCK_122 runs both on the same files.
#
# ADDED FOR TRACK B, and not in the counter:
#   name_of()   a conversation's name for the split: "conversations/" + the SHA-256 of its rendering, so that
#               identical conversations share a name, and so a side, and Track A's split rule (one label) applies;
#   scored()    the token positions a fine-tune learns and the scorer scores: each assistant turn's content and
#               its <|im_end|>, from the whole conversation tokenized once.
import hashlib, io, json, re
ROW = "train-llm:qwen3-4b-instruct-2507"
COUNT_VERSION = "nexido-count-v1"
RENDER_RULE = "chatml-plain-v1"
TOKENIZER_SHA256 = "aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4"
TOKENIZER_ENTRIES = 151669
# count-receipt.ts COUNT_ROWS for this row, decided 2026-10-08 (ADR-0066 decisions 3 and 4)
LIMITS = {"max_bytes": 32 * 1024 * 1024, "max_rows": 200000, "max_row_bytes": 65536, "max_tokens_per_row": 2048}
ROLES = ("system", "user", "assistant")
NAME_PREFIX = b"conversations/"
BATCH_ROWS, BATCH_CHARS = 256, 65536
class Refusal(Exception):
    """A file the counter would refuse. row: 1-based, blank lines not counted; line: 1-based, blank lines
    counted; both None for the whole file."""
    def __init__(self, reason, row, line, detail):
        Exception.__init__(self, "%s (row %s, line %s): %s" % (reason, row, line, detail))
        self.reason, self.row, self.line, self.detail = reason, row, line, detail
class RowRefusal(Exception):
    def __init__(self, reason, detail):
        Exception.__init__(self, reason)
        self.reason, self.detail = reason, detail
def load_tokenizer(path):
    """The pinned tokenizer and its control-token texts (every added token), or ValueError. Fails closed."""
    from tokenizers import Tokenizer
    b = io.open(path, "rb").read()
    got = hashlib.sha256(b).hexdigest()
    if got != TOKENIZER_SHA256:
        raise ValueError("%s has sha256 %s, expected %s" % (path, got, TOKENIZER_SHA256))
    tok = Tokenizer.from_str(b.decode("utf-8"))
    if tok.get_vocab_size(with_added_tokens=True) != TOKENIZER_ENTRIES:
        raise ValueError("the tokenizer has %d entries, expected %d" % (tok.get_vocab_size(True), TOKENIZER_ENTRIES))
    control = []
    for i in range(tok.get_vocab_size(with_added_tokens=False), TOKENIZER_ENTRIES):
        t = tok.id_to_token(i)
        if not isinstance(t, str) or t == "":
            raise ValueError("token %d has no text" % i)
        control.append(t)
    return tok, control
def _shown(key):
    return json.dumps(key[:40] + "..." if len(key) > 40 else key)
def count_keys(s):
    """How many object keys a line of VALID JSON holds: a string followed by a colon. As the counter's."""
    keys, i, n = 0, 0, len(s)
    while i < n:
        if s[i] != '"':
            i += 1
            continue
        i += 1
        while i < n and s[i] != '"':
            i += 2 if s[i] == "\\" else 1
        i += 1
        while i < n and s[i] in " \t\r\n":
            i += 1
        if i < n and s[i] == ":":
            keys += 1
    return keys
_WS = re.compile(r"[ \t\n\r]*")
_STR = re.compile(r'"(?:[^"\\\x00-\x1f]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*"')
_NUM = re.compile(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")
_LIT = (("true", True), ("false", False), ("null", None))
def _loads_flat(s):
    """JSON.parse's grammar without recursion, for a row nested deeper than json.loads may go. JavaScript parses
    such a row and then names a structural reason; Python's parser stops on its recursion limit instead. Strings
    are decoded by json.loads itself; containers below depth 4 come back empty, since parse_row reads nothing
    there and a row that holds one is refused whatever it contains."""
    def bad(i):
        raise ValueError("not JSON at %d" % i)
    n, stack, root = len(s), [], None
    i = _WS.match(s, 0).end()
    while True:
        # a value starts at i
        if i >= n:
            bad(i)
        c = s[i]
        if c in "{[":
            box = {} if c == "{" else []
            stack.append([box, None, len(stack) + 1])
            i = _WS.match(s, i + 1).end()
            if i < n and s[i] == ("}" if c == "{" else "]"):
                val, i = stack.pop()[0], i + 1
            elif c == "{":
                m = _STR.match(s, i)
                if not m:
                    bad(i)
                stack[-1][1] = json.loads(m.group())
                i = _WS.match(s, m.end()).end()
                if i >= n or s[i] != ":":
                    bad(i)
                i = _WS.match(s, i + 1).end()
                continue
            else:
                continue
        elif c == '"':
            m = _STR.match(s, i)
            if not m:
                bad(i)
            val, i = json.loads(m.group()), m.end()
        else:
            m = _NUM.match(s, i)
            if m and m.end() > i:
                val, i = 0, m.end()
            else:
                for t, v in _LIT:
                    if s.startswith(t, i):
                        val, i = v, i + len(t)
                        break
                else:
                    bad(i)
        # a value ended at i: place it, then close every container that ends here
        while True:
            if not stack:
                if _WS.match(s, i).end() != n:
                    bad(i)
                return root if root is not None else val
            box, key, depth = stack[-1]
            if depth <= 4:
                if isinstance(box, dict):
                    box[key] = val
                else:
                    box.append(val)
            i = _WS.match(s, i).end()
            if i < n and s[i] == ",":
                i = _WS.match(s, i + 1).end()
                if isinstance(box, dict):
                    m = _STR.match(s, i)
                    if not m:
                        bad(i)
                    stack[-1][1] = json.loads(m.group())
                    i = _WS.match(s, m.end()).end()
                    if i >= n or s[i] != ":":
                        bad(i)
                    i = _WS.match(s, i + 1).end()
                break
            if i < n and s[i] == ("}" if isinstance(box, dict) else "]"):
                val, i = stack.pop()[0], i + 1
                if not stack:
                    root = val
                continue
            bad(i)
def _bad_constant(c):
    raise ValueError("%s is not JSON" % c)
def _int_zero(digits):
    # JSON.parse reads an integer of any length; Python refuses one of more than 4,300 digits (3.10.7 and later).
    # No check reads a number's value, only that it is not a string, so every integer becomes 0.
    return 0
def parse_row(text, control):
    """One line of the file -> its messages as (role, content) pairs, or RowRefusal. As the counter's parseRow,
    check for check and in the same order."""
    try:
        try:
            doc = json.loads(text, parse_constant=_bad_constant, parse_int=_int_zero)
        except RecursionError:
            doc = _loads_flat(text)
    except (ValueError, RecursionError):
        raise RowRefusal("invalid_json", "the line is not valid JSON")
    if not isinstance(doc, dict):
        raise RowRefusal("not_an_object", "a row must be one JSON object")
    for k in doc:
        if k != "messages":
            raise RowRefusal("unknown_key", 'unknown key %s; a row holds "messages" only' % _shown(k))
    raw = doc.get("messages")
    if not isinstance(raw, list) or len(raw) == 0:
        raise RowRefusal("no_messages", '"messages" must be a non-empty array')
    messages = []
    for m in raw:
        if not isinstance(m, dict):
            raise RowRefusal("bad_message", "every message must be a JSON object")
        for k in m:
            if k != "role" and k != "content":
                raise RowRefusal("unknown_key", "unknown message key %s" % _shown(k))
        role, content = m.get("role"), m.get("content")
        if not isinstance(role, str) or role not in ROLES:
            raise RowRefusal("bad_role", 'role must be "system", "user" or "assistant"')
        if not isinstance(content, str):
            raise RowRefusal("bad_content", "content must be a string")
        if any(0xD800 <= ord(ch) <= 0xDFFF for ch in content):
            raise RowRefusal("bad_content", "content holds an unpaired surrogate")
        if content == "":
            raise RowRefusal("empty_content", "the %s message is empty" % role)
        for t in control:
            if t in content:
                raise RowRefusal("control_token", "content holds the control token text %s" % t)
        messages.append((role, content))
    if count_keys(text) != 1 + 2 * len(messages):
        raise RowRefusal("duplicate_key", "a key appears twice in the row; only the last would have been read")
    turns = messages[1:] if messages[0][0] == "system" else messages
    if not (len(turns) >= 2 and len(turns) % 2 == 0 and all(
            r == ("user" if i % 2 == 0 else "assistant") for i, (r, _) in enumerate(turns))):
        raise RowRefusal("bad_turn_order", "expected [system,] user, assistant, ... ending on assistant")
    return messages
def render(messages):
    """Plain ChatML, the counter's renderChatml."""
    return "".join("<|im_start|>%s\n%s<|im_end|>\n" % (r, c) for r, c in messages)
def _blank(b):
    return all(c in (0x20, 0x09, 0x0D) for c in b)
def read_file(data, tok, control, limits=LIMITS, keep=False):
    """The counter's countTrainingFile over a file's bytes (pass at most max_bytes + 1 of them). Returns a dict
    with rows, tokens, row_tokens, bytes, file_sha256, row_tokens_sha256, and, with keep, every conversation as
    (messages, rendering, row, line); or raises Refusal, naming the first row in file order that breaks a rule."""
    mb, mr, mrb, mt = (limits[k] for k in ("max_bytes", "max_rows", "max_row_bytes", "max_tokens_per_row"))
    for v in (mb, mr, mrb, mt):
        if type(v) is not int or v < 1:
            raise ValueError("%r is not a usable limit" % (v,))
    over, body = len(data) > mb, data[:mb]
    st = {"rows": 0, "lines": 0}
    row_tokens, convs, pending = [], [], []
    def flush():
        if pending:
            texts = [p[1] for p in pending]
            got = [len(e.ids) for e in tok.encode_batch(texts, add_special_tokens=False)]
            if len(got) != len(texts):
                raise RuntimeError("asked for %d counts, received %d" % (len(texts), len(got)))
            for (msgs, text, row, line), n in zip(pending, got):
                if n > mt:
                    del pending[:]
                    raise Refusal("row_too_long", row, line, "%d tokens exceeds the %d token limit" % (n, mt))
                row_tokens.append(n)
                if keep:
                    convs.append((msgs, text, row, line))
            del pending[:]
    def stop(reason, row, line, detail):
        flush()
        raise Refusal(reason, row, line, detail)
    def take_line(b):
        st["lines"] += 1
        if st["lines"] == 1 and b[:3] == b"\xef\xbb\xbf":
            b = b[3:]
        if _blank(b):
            return
        st["rows"] += 1
        rows, lines = st["rows"], st["lines"]
        if rows > mr:
            stop("too_many_rows", rows, lines, "more than %d rows" % mr)
        try:
            text = b.decode("utf-8")
        except UnicodeDecodeError:
            stop("invalid_utf8", rows, lines, "the row is not valid UTF-8")
        try:
            msgs = parse_row(text, control)
        except RowRefusal as r:
            stop(r.reason, rows, lines, r.detail)
        rendered = render(msgs)
        pending.append((msgs, rendered, rows, lines))
        if len(pending) >= BATCH_ROWS or sum(len(p[1]) for p in pending) >= BATCH_CHARS:
            flush()
    start = 0
    while True:
        nl = body.find(b"\n", start)
        if nl < 0:
            break
        piece, start = body[start:nl], nl + 1
        if len(piece) > mrb:
            stop("row_too_large", st["rows"] + 1, st["lines"] + 1,
                 "the line is longer than the %d byte limit for one row" % mrb)
        take_line(piece)
    rest = body[start:]
    if len(rest) > mrb:
        stop("row_too_large", st["rows"] + 1, st["lines"] + 1,
             "the line is longer than the %d byte limit for one row" % mrb)
    if over:
        stop("too_large", None, None, "more than %d bytes" % mb)
    if rest:
        take_line(rest)
    flush()
    if st["rows"] == 0:
        raise Refusal("empty_input", None, None, "the file holds no rows")
    return {"rows": st["rows"], "tokens": sum(row_tokens), "row_tokens": row_tokens, "bytes": len(body),
            "file_sha256": hashlib.sha256(body).hexdigest(),
            "row_tokens_sha256": hashlib.sha256(",".join(str(n) for n in row_tokens).encode("ascii")).hexdigest(),
            "conversations": convs}
def name_of(rendered):
    """A conversation's name for the split rule: one label, "conversations"; the rest is the rendering's hash."""
    return NAME_PREFIX + hashlib.sha256(rendered.encode("utf-8")).hexdigest().encode("ascii")
def answer_spans(messages):
    """Character spans of each assistant turn's content and its <|im_end|> in the rendering."""
    spans, pos = [], 0
    for role, content in messages:
        a = pos + len("<|im_start|>%s\n" % role)
        b = a + len(content) + len("<|im_end|>")
        if role == "assistant":
            spans.append((a, b))
        pos = b + 1
    return spans
def scored(offsets, messages):
    """The positions of the tokens wholly inside an assistant span (never the first token), and how many tokens
    straddle a span's edge; those are not scored, and the trainer must not learn them either."""
    sp = answer_spans(messages)
    ans, cut = [], 0
    for t, (x, y) in enumerate(offsets):
        if t > 0 and any(a <= x and y <= b for a, b in sp):
            ans.append(t)
        cut += any(x < a < y or x < b < y for a, b in sp)
    return ans, cut
