# NEXIDO -- INDEPENDENT REPRODUCTION OF A TRACK A SCORE
#
# Point it at a directory holding one run's features, head and score artifacts. It re-derives the
# test split by the published rule, re-applies the trained head, and checks the predictions it got
# against the ones the score artifact carries -- by digest, so one flipped prediction fails.
# It then counts the byte-identical images shared by the training and test sets (step 9).
# Needs numpy and split_rule.py. No GPU, no network, no Nexido account.
#     python3 reproduce.py DIR [DIR ...]
import hashlib
import io, json, os, sys
import numpy as np

# split_rule.py is the published split rule and the only Nexido file this needs. It
# normally sits beside this script; NEXIDO_SPLIT_RULE overrides that if you keep it
# somewhere else.
sys.path.insert(0, os.environ.get('NEXIDO_SPLIT_RULE')
                or os.path.dirname(os.path.abspath(__file__)))
try:
    import split_rule
except ImportError:
    raise SystemExit('split_rule.py must sit beside this script, or NEXIDO_SPLIT_RULE '
                     'must name the directory holding it.')

MAGIC = {b'NEXIDOFEAT\x01': 'features', b'NEXIDOHEAD\x01': 'head', b'NEXIDOSCOR\x01': 'score'}
NPDT = {'float32': '<f4', 'int32': '<i4'}
BREAK = os.environ.get('BREAK', '')
HOME = os.environ.get('HOME', '')
DUPS = {}


def read_block(path):
    raw = io.open(path, 'rb').read()
    kind = MAGIC.get(raw[:11])
    if kind is None:
        return None
    try:
        hlen = int(raw[11:21].decode('ascii'))
    except Exception:
        return None
    if raw[21:22] != b'\n':
        return None
    return {'kind': kind, 'hdr': json.loads(raw[22:22 + hlen].decode('utf-8')),
            'pay': raw[22 + hlen:], 'name': os.path.basename(path),
            'file_sha': hashlib.sha256(raw).hexdigest(),
            'pay_sha': hashlib.sha256(raw[22 + hlen:]).hexdigest()}


def dg(h, k):
    return str(h.get(k, '')).replace('sha256:', '')


def skip(m):
    print('  SKIP -- %s' % m)


def no(m):
    print('  *** %s' % m)


def npdtype(h, who):
    d, bo = h.get('dtype'), h.get('byte_order')
    if bo != 'little' or d not in NPDT:
        no('%s declares dtype %r byte_order %r, not implemented by this reader' % (who, d, bo))
        return None
    return NPDT[d]


def duplicate_check(d, names, shas, assignment, declared):
    # EXACT-DUPLICATE CHECK. The same arithmetic as the leakage block in run_score.py, computed
    # here from the per-image sha256 the features entries carry and the split select() proved.
    # It sees a BYTE-IDENTICAL file in both sets and nothing else, and it says so.
    print()
    if declared is not None and not isinstance(declared, dict):
        declared = {}
    if len(shas) != len(names) or not all(isinstance(s, str) and s for s in shas):
        DUPS[d] = 'NOT COMPUTED -- the features entries carry no per-image sha256'
        print('  9. duplicate check NOT COMPUTED: the features entries carry no per-image sha256.')
        if declared is not None:
            no('the score declares a leakage result that these entries cannot support')
        return declared is None

    def count(sh):
        rows = {}
        for key in ('train', 'val', 'test'):
            members, got = set(assignment[key]), {}
            for i, nm in enumerate(names):
                if nm.encode('utf-8') in members:
                    got.setdefault(sh[i], []).append(i)
            rows[key] = got
        tr, te = rows['train'], rows['test']
        col = sorted(set(tr) & set(te))
        n_tr = sum(len(v) for v in tr.values())
        n_te = sum(len(v) for v in te.values())
        return rows, col, {
            'checked_rows_train': n_tr, 'checked_rows_test': n_te,
            'exact_duplicate_digests': len(col),
            'exact_duplicate_test_rows': sum(len(te[h]) for h in col),
            'exact_duplicate_train_rows': sum(len(tr[h]) for h in col),
            'duplicate_rows_within_train': n_tr - len(tr),
            'duplicate_rows_within_test': n_te - len(te)}

    rows, col, got = count(shas)
    clean = dict(got)
    if BREAK == 'leak':
        free = [i for h in sorted(rows['test']) if h not in rows['train'] for i in rows['test'][h]]
        donors = sorted(rows['train'])
        if free and donors:
            planted = list(shas)
            planted[free[0]] = donors[0]
            rows, col, got = count(planted)
        print('  9. *** BREAK=leak: one test row was given a training row digest on purpose.')
    tr, va, te = rows['train'], rows['val'], rows['test']
    print('  9. duplicate check, from the per-image sha256 in the features entries and the split')
    print('     re-derived above: %d train rows against %d test rows'
          % (got['checked_rows_train'], got['checked_rows_test']))
    print('       %-56s %d' % ('byte-identical digests present in BOTH train and test',
                              got['exact_duplicate_digests']))
    print('       %-56s %d of %d' % ('test rows that duplicate a training image',
                                    got['exact_duplicate_test_rows'], got['checked_rows_test']))
    print('       %-56s %d' % ('training rows they duplicate', got['exact_duplicate_train_rows']))
    print('       %-56s %d / %d' % ('repeats inside train alone / inside test alone',
                                   got['duplicate_rows_within_train'],
                                   got['duplicate_rows_within_test']))
    print('       %-56s %d   (not part of the recorded result)'
          % ('test rows that duplicate a VALIDATION image',
             sum(len(te[h]) for h in te if h in va)))
    if col:
        per = {}
        for h in col:
            for i in te[h]:
                lab = names[i].split('/')[-2] if '/' in names[i] else '?'
                per[lab] = per.get(lab, 0) + 1
        print('       by class, test rows: %s'
              % ', '.join('%s %d' % (k, per[k]) for k in sorted(per, key=lambda k: (-per[k], k))[:8]))
        for h in col[:3]:
            print('       e.g. %s (test) is the same file as %s (train)'
                  % (names[te[h][0]][:60], names[tr[h][0]][:60]))
    ok = True
    if declared is None:
        print('     The score artifact declares no leakage result -- its scorer predates the field --')
        print('     so these figures are computed here, from your files alone.')
    else:
        diff = [k for k in sorted(got) if declared.get(k) != got[k]]
        if diff:
            ok = False
            no('the score artifact declares a different leakage result: %s is %r there, %r here'
               % (diff[0], declared.get(diff[0]), got[diff[0]]))
        else:
            print('     The score artifact declares the same seven figures: yes')
    print('     NOT visible to this check: a resized, recompressed or re-encoded copy of a test')
    print('     image, and the same subject photographed again. Neither is byte-identical.')
    DUPS[d] = ('none found' if not got['exact_duplicate_digests'] else
               'FOUND -- %d test row(s) byte-identical to a training image'
               % got['exact_duplicate_test_rows'])
    if BREAK == 'leak':
        if got != clean:
            print('     The planted duplicate moved the count from %d to %d, so the check sees it.'
                  % (clean['exact_duplicate_test_rows'], got['exact_duplicate_test_rows']))
            return False
        print('     *** The planted duplicate did NOT move the count. The check is blind.')
    return ok


def reproduce(d):
    print()
    print('=' * 94)
    print('DIRECTORY  %s' % d.replace(HOME, '~'))
    blocks = []
    for fn in sorted(os.listdir(d)):
        p = os.path.join(d, fn)
        if os.path.isfile(p) and os.path.getsize(p) >= 23:
            b = read_block(p)
            if b:
                blocks.append(b)
    feats = [b for b in blocks if b['kind'] == 'features']
    heads = [b for b in blocks if b['kind'] == 'head']
    scores = [b for b in blocks if b['kind'] == 'score']
    print('  found: %d features, %d head, %d score' % (len(feats), len(heads), len(scores)))
    if not feats or not heads or not scores:
        skip('not a complete triple.')
        return None
    bad = [b['name'] for b in blocks if b['pay_sha'] != dg(b['hdr'], 'payload_sha256')]
    if bad:
        skip('payload does not verify for: %s. A corrupted copy that still parses answers '
             'confidently and wrongly.' % ', '.join(bad[:6]))
        return None
    print('  1. every payload verifies against its own declared digest : %d of %d'
          % (len(blocks), len(blocks)))

    sc = scores[0]
    sh = sc['hdr']
    want = dg(sh, 'head_sha256')
    hd = None
    for h in heads:
        if h['file_sha'] == want:
            hd = h
    if hd is None:
        skip('no head here has the FILE digest the score declares, %s...' % want[:16])
        return None
    hh = hd['hdr']
    print('  2. score declares head_sha256 %s... and %s is that exact file' % (want[:16], hd['name']))

    inputs = sh.get('inputs')
    if not isinstance(inputs, list) or not inputs:
        skip('this score carries no inputs list, so its features cannot be identified.')
        return None
    bysha = {}
    for f in feats:
        bysha[f['file_sha']] = f
    chosen = []
    for inp in inputs:
        w = dg(inp, 'artifact_sha256')
        if w not in bysha:
            skip('score names features file %s... which is not in this directory.' % w[:16])
            return None
        g = bysha[w]
        if g['hdr'].get('slice_index') != inp.get('slice_index'):
            no('input says slice_index %r, the file says %r'
               % (inp.get('slice_index'), g['hdr'].get('slice_index')))
            return False
        chosen.append(g)
    print('  3. all %d declared input digests match features FILES here, slice_index agreeing'
          % len(chosen))

    chosen.sort(key=lambda b: b['hdr']['slice_index'])
    one = chosen[0]['hdr']
    DIM = one['dim']
    fdt = npdtype(one, 'the features artifact')
    if fdt is None:
        return False
    for k in ('dim', 'dtype', 'byte_order', 'slice_count', 'archive_sha256', 'model_sha256',
              'total_images_in_archive'):
        vals = set()
        for b in chosen:
            vals.add(json.dumps(b['hdr'].get(k), sort_keys=True))
        if len(vals) != 1:
            no('the slices disagree on %s, so they are not one archive' % k)
            return False
    idxs = sorted([b['hdr']['slice_index'] for b in chosen])
    if idxs != list(range(one['slice_count'])):
        no('slice indices %s are not a complete 0..%d run' % (str(idxs)[:60], one['slice_count'] - 1))
        return False
    mats, names, shas, rows = [], [], [], 0
    for b in chosen:
        r = b['hdr']['rows']
        if len(b['pay']) != r * DIM * 4:
            no('%s payload is %d bytes, not rows*dim*4 = %d' % (b['name'], len(b['pay']), r * DIM * 4))
            return False
        mats.append(np.frombuffer(b['pay'], dtype=fdt).reshape(r, DIM))
        names.extend([e['name'] for e in b['hdr']['entries']])
        shas.extend([e.get('sha256') for e in b['hdr']['entries']])
        rows += r
    if rows != one['total_images_in_archive']:
        no('slices sum to %d rows, the archive declares %d images' % (rows, one['total_images_in_archive']))
        return False
    if len(names) != rows:
        no('%d entry names for %d rows; row alignment cannot be assumed' % (len(names), rows))
        return False
    X_all = np.concatenate(mats, axis=0).astype(np.float64)       # run_score.py:139
    print('  4. features loaded as %s: %d slices, X_all %s, %d names in ROW order, and the'
          % (fdt, len(chosen), X_all.shape, len(names)))
    print('     archive declares %d images which is exactly what the slices sum to' % rows)

    if hh.get('model_sha256') != sh.get('model_sha256'):
        no('the head and the score name different feature models')
        return False
    classes = sh.get('classes')
    if classes != hh.get('classes'):
        no('the score and the head disagree about the class list')
        return False
    if not isinstance(classes, list) or not classes:
        no('classes is %r, not a non-empty list' % type(classes).__name__)
        return False
    rule = sh.get('label_rule')
    if rule != 'parent':
        skip('label_rule is %r; this reader implements "parent" only and will not guess.' % rule)
        return None
    nh, P, C = hh['heads_shape']
    if P != DIM + 1 or C != len(classes) or nh != sh.get('heads_scored'):
        no('heads_shape %r disagrees with dim %d, %d classes, or heads_scored %r'
           % (hh['heads_shape'], DIM, len(classes), sh.get('heads_scored')))
        return False
    print('  5. head and score agree: one model, %d classes, heads_shape %r, label_rule %r'
          % (C, hh['heads_shape'], rule))

    split = sh['split']
    if BREAK == 'seed':
        split = dict(split)
        split['seed'] = split['seed'] + 1
        print('  6. *** BREAK=seed: the declared seed has been altered by +1 on purpose.')
    try:
        assignment = split_rule.select([n.encode('utf-8') for n in names], split)
    except Exception as e:
        print('  6. split_rule.select REFUSED: %s: %s'
              % (type(e).__name__, str(getattr(e, 'reason', e))[:80]))
        print('     The published rule doing its job. Nothing downstream is trustworthy.')
        return False
    s_test = set(assignment['test'])                              # run_score.py:231-233
    idx_test = [i for i, nm in enumerate(names) if nm.encode('utf-8') in s_test]
    print('  6. split re-derived from seed %s under rule_version %s; select() proved all three'
          % (split['seed'], split['rule_version']))
    print('     declared digests itself. test rows %d, header n_test %s'
          % (len(idx_test), sh.get('n_test')))
    if len(idx_test) != sh.get('n_test'):
        no('that disagrees with the header. Stopping rather than reporting a number.')
        return False

    cidx = {}
    for i, c in enumerate(classes):
        cidx[c] = i
    ys = []
    for i in idx_test:
        parts = names[i].split('/')
        lab = parts[-2] if len(parts) > 1 else None
        if not lab or lab not in cidx:
            no('row %d name %r yields label %r, not one of the artifact classes'
               % (i, names[i][:60], lab))
            return False
        ys.append(cidx[lab])
    ys = np.asarray(ys, dtype=np.int64)
    print('  7. labels taken by the "parent" rule from the stored names; all %d are known classes'
          % len(ys))

    hdt = npdtype(hh, 'the head artifact')
    if hdt is None:
        return False
    if len(hd['pay']) != nh * P * C * 4:
        no('head payload is %d bytes, not nh*P*C*4 = %d' % (len(hd['pay']), nh * P * C * 4))
        return False
    Wall = np.frombuffer(hd['pay'], dtype=hdt).reshape(nh, P, C)
    Xs = X_all[idx_test]
    n = Xs.shape[0]
    Xb = np.empty((n, P), dtype=np.float64)                       # run_score.py:268
    Xb[:, :DIM] = Xs                                              # run_score.py:269
    Xb[:, DIM] = 1.0                                              # run_score.py:270
    if BREAK == 'bias':
        Xb[:, DIM] = 0.0
        print('  8. *** BREAK=bias: the bias column is 0.0 instead of 1.0 on purpose.')
    pmat = np.empty((nh, n), dtype=np.int32)
    for h in range(nh):
        L = Xb @ Wall[h]                                          # run_score.py:274
        pmat[h] = np.argmax(L, axis=1).astype(np.int32)            # run_score.py:277
    if BREAK == 'flip':
        pmat[0, 0] = (pmat[0, 0] + 1) % C
        print('  8. *** BREAK=flip: ONE prediction of %d has been changed on purpose.' % (nh * n))
    print('  8. head applied as %s: %d heads x %d test rows, bias column a literal 1.0'
          % (hdt, nh, n))

    mine = hashlib.sha256(np.ascontiguousarray(pmat, dtype='<i4').tobytes()).hexdigest()
    exact = mine == sc['pay_sha']
    print()
    print('  %-28s %s' % ('predictions I computed', mine))
    print('  %-28s %s' % ('predictions in the artifact', sc['pay_sha']))
    print('  %-28s %s' % ('BIT-EXACT MATCH', 'YES' if exact else '*** NO'))
    if not exact:
        try:
            tp = np.frombuffer(sc['pay'], dtype='<i4').reshape(nh, n)
            df = np.nonzero(pmat != tp)
            print('  %-28s %d of %d, first at head %d row %d: mine %d theirs %d'
                  % ('differing predictions', len(df[0]), nh * n, df[0][0], df[1][0],
                     pmat[df[0][0], df[1][0]], tp[df[0][0], df[1][0]]))
        except Exception as e:
            print('  could not decode theirs for comparison: %s' % str(e)[:80])

    per = sh.get('per_head') or []
    nchk = min(nh, len(per))
    print()
    print('  %-9s %-14s %-12s %-16s %s'
          % ('lambda', 'their correct', 'my correct', 'confusion', 'agree?'))
    nagree = 0
    for h in range(nchk):
        cm = np.zeros((C, C), dtype=np.int64)
        for i in range(n):
            cm[ys[i], pmat[h, i]] += 1                            # run_score.py:283-284
        corr = int(np.trace(cm))                                   # run_score.py:285
        tc, tcm = per[h].get('correct'), per[h].get('confusion')
        cm_ok = isinstance(tcm, list) and cm.tolist() == tcm
        ok = corr == tc and cm_ok
        nagree += 1 if ok else 0
        print('  %-9s %-14s %-12s %-16s %s'
              % (hh['lambda_grid'][h], tc, corr,
                 '%dx%d exact' % (C, C) if cm_ok else '*** DIFFERS', 'yes' if ok else '*** NO'))
    print()
    print('  heads whose correct count AND full confusion matrix agree exactly : %d of %d'
          % (nagree, nchk))
    dup_ok = duplicate_check(d, names, shas, assignment, sh.get('leakage'))
    return exact and nagree == nchk and nchk == nh and dup_ok


results = []
for d in sys.argv[1:]:
    if os.path.isdir(d):
        results.append((d, reproduce(d)))
    else:
        print('  ABSENT  %s' % d.replace(HOME, '~'))
ok = [d for d, r in results if r is True]
bad_ = [d for d, r in results if r is False]
sk = [d for d, r in results if r is None]
print()
print('=' * 94)
print('VERDICT   reproduced %d   did NOT reproduce %d   skipped %d' % (len(ok), len(bad_), len(sk)))
for d in bad_:
    print('    did not reproduce: %s' % d.replace(HOME, '~'))
for d in sk:
    print('    skipped: %s' % d.replace(HOME, '~'))
for d, r in results:
    if d in DUPS:
        print('    duplicate check, %s: %s' % (d.replace(HOME, '~'), DUPS[d]))
if BREAK:
    print('  THIS WAS A BREAK RUN, BREAK=%s. Anything reproducing here would mean the check' % BREAK)
    print('  cannot detect that fault, which would make the clean run worthless.')
elif ok and not bad_ and not sk:
    print('  >>> EXIT CRITERION 10.5, SECOND HALF, on %d independent run(s): the artifact loads'
          % len(ok))
    print('      outside Nexido, on numpy and the published split rule alone, and reproduces EVERY')
    print('      prediction it reported -- by digest, with every confusion matrix matching element')
    print('      for element.')
else:
    print('  *** NOT A PASS. Read the per-directory output above.')
raise SystemExit(0 if ok and not bad_ and not sk else 1)
