import json, gzip, glob, bisect, collections, math, statistics, sys
sys.path.insert(0, '/tmp/rl')
from tl import eth, KL, RT, SUP, QDEC, Z, WETH, USDG, CHK, LEVELS, qusd_direct
import pandas as pd
cal = collections.defaultdict(list)
for f in glob.glob('/tmp/rl/calib_*.json'):
    for q, d, r in json.load(open(f)): cal[q].append(r)
CAL = {q: statistics.median(v) for q, v in cal.items() if len(v) >= 2}
def qusd(q, t):
    u = qusd_direct(q, t)
    if u: return u, 'direct'
    if q in CAL: return CAL[q], 'calib'
    return None, None

def tier(v):
    if v is None: return None
    for k, t in enumerate((5e5, 1e6, 2e6, 5e6, 2e7)[::-1]):
        if v >= t: return 5 - k
    return 0

rows = []; skip = collections.Counter(); WALL = {}
for f in sorted(glob.glob('/tmp/rl/rows_*.jsonl.gz')):
    for line in gzip.open(f, 'rt'):
        o = json.loads(line); tok = o['tok']; t0 = o['t0']; q = o['mq']
        u0, src = qusd(q, t0)
        if not u0: skip['no_qusd'] += 1; continue
        dec = int(o['l']['dec'] if o['l']['dec'] is not None else 18); qd = (QDEC.get(q) or [18])[0] or 18
        sup = o['sup']
        ev = [(t, m, px, -qa, tx, qq) for t, m, px, qa, tx, qq in o['ev']]   # stored qa is user side; flip so >0 = buy
        mev = [(t, px) for t, m, px, qa, tx, qq in ev if m]
        if len(mev) < 2: skip['few_main'] += 1; continue
        def u(t):
            x = qusd(q, t0 + t)[0]; return x if x else u0
        # USD per event (all pools; other-pool quotes converted with their own price)
        def evusd(qq, qa, t):
            x = qusd(qq, t0 + t)[0]
            return qa * x if x else None
        r = dict(tok=tok, day=o['day'], t0=t0, quote=q, qsrc=src, hook=o['hook'], fee=o['fee'], npools=o['npools'],
                 fac=o['l']['fac'], cr=o['l']['cr'], name=o['l']['nm'], sym=o['l']['sym'],
                 wait_pool=(t0 - o['l']['ts']) if o['l']['ts'] else None)
        # confirmed mcap path (min of consecutive prices)
        mt = [t for t, px in mev]; mc = [px * sup * u(t) for t, px in mev]
        conf = [min(mc[i], mc[i + 1]) for i in range(len(mc) - 1)]
        cmax = []; m = 0
        for v in conf: m = max(m, v); cmax.append(m)
        r['mc_first'] = mc[0]
        for L in [5e4, 1e5, 2.5e5, 5e5, 1e6, 2e6, 5e6]:
            i = next((k for k, v in enumerate(conf) if v >= L), None)
            r['tx_%g' % L] = mt[i + 1] if i is not None else None
        buys = o.get('buys') or []
        evs = [(t, qa, qq) for t, m_, px, qa, tx, qq in ev]
        for T in CHK:
            i = bisect.bisect_right(mt, T) - 1
            if i < 0: r['mc_%d' % T] = None; continue
            r['mc_%d' % T] = mc[i]
            r['mcmax_%d' % T] = cmax[i - 1] if i >= 1 else mc[0]
            e = [x for x in evs if x[0] <= T]
            r['nb_%d' % T] = sum(1 for x in e if x[1] > 0); r['ns_%d' % T] = sum(1 for x in e if x[1] < 0)
            bu = [evusd(qq, qa, t) for t, qa, qq in e if qa > 0]; su = [evusd(qq, -qa, t) for t, qa, qq in e if qa < 0]
            r['bu_%d' % T] = sum(x for x in bu if x); r['su_%d' % T] = sum(x for x in su if x)
            if T <= 1800:
                bb = [x for x in buys if x[0] <= T]
                ws = collections.Counter()
                for t, a, v in bb: ws[a] += int(v)
                r['ub_%d' % T] = len(ws)
                tot = sum(ws.values()) or 1
                r['top1b_%d' % T] = max(ws.values()) / tot if ws else None
                r['top5b_%d' % T] = sum(v for _, v in ws.most_common(5)) / tot if ws else None
        # outcome
        pk_rt = None
        if tok in KL and (SUP.get(tok) or 0) > 0:
            w = [x for x in KL[tok] if t0 - 3600 <= x[0] <= t0 + 7 * 86400]
            if w:
                cl = sorted((x[4] * SUP[tok] for x in w), reverse=True)
                pk_rt = cl[1] if len(cl) > 1 else cl[0]
        pk_sw = max(conf) if conf else None
        sells = [(t, px) for t, m_, px, qa, tx, qq in ev if m_ and qa < 0 and (evusd(qq, -qa, t) or 0) >= 100]
        smc = [(t, px * sup * u(t)) for t, px in sells]
        r['pk_sell'] = max((v for t, v in smc), default=None)
        r['pk_sell_t'] = max(smc, key=lambda z: z[1])[0] if smc else None
        for T in CHK:
            r['pa_%d' % T] = max((v for t, v in smc if t > T), default=0)
        for L in [1e5, 2.5e5, 5e5, 1e6, 2e6, 5e6]:
            r['ts_%g' % L] = next((t for t, v in smc if v >= L), None)
        bu_all = [(t, evusd(qq, qa, t)) for t, m_, px, qa, tx, qq in ev if qa > 0]
        r['first_buy_usd'] = (bu_all[0][1] or 0) if bu_all else 0
        r['buys_2s_usd'] = sum((v or 0) for t, v in bu_all if t <= 2)
        W60 = sorted({a for t, a, v in buys if t <= 60}); W300 = sorted({a for t, a, v in buys if t <= 300})
        WALL[tok] = (W60, W300)
        r['rt_created'] = RT[tok].get('created_ts') if tok in RT else None
        bb = o['bars']; pk_b = None
        if len(bb) >= 2:
            inv = o.get('inv')
            ps = []
            for t, cl in bb:
                if cl and cl > 0:
                    px = (1 / cl if inv else cl) * 10 ** (dec - qd); ps.append((t, px * sup * u(t)))
            if len(ps) >= 2: pk_b = max(min(ps[i][1], ps[i + 1][1]) for i in range(len(ps) - 1))
        r['pk_rt'] = pk_rt; r['pk_sw'] = pk_sw; r['pk_b'] = pk_b; r['in_rt'] = tok in RT
        r['peak'] = pk_rt if pk_rt is not None else (r['pk_sell'] or 0)
        r['tier'] = tier(r['peak'])
        r['rt_cross'] = (RT[tok].get('crossed_ts') if tok in RT else None)
        rows.append(r)
df = pd.DataFrame(rows)
df.to_pickle('/tmp/rl/df.pkl'); json.dump(WALL, gzip.open('/tmp/rl/wall.json.gz','wt'))
print(skip, len(df))
print(df.tier.value_counts().sort_index())
print('quote src', df.qsrc.value_counts())
print('coverage: not in runtrack but swap/bar peak >= 1M:', ((~df.in_rt) & (df.peak >= 1e6)).sum(), ' in rt:', (df.in_rt).sum())
print('in rt but rt peak < 500k:', ((df.in_rt) & (df.pk_rt.fillna(0) < 5e5)).sum())