#!/usr/bin/env python3
"""Ticker crowd in the 10h before a new-ATH breakout vs the 24h before that. SUCCESS vs FAKE."""
import json, glob, re, random, statistics as st, collections, math
EV = {f'{e["address"]}:{e["t"]}': e for e in json.load(open("/opt/bostudy/events.json"))}
X = {}
for fn in glob.glob("/opt/bostudy/xraw*.jsonl"):
    for l in open(fn):
        try: r = json.loads(l); X[r["k"]] = r
        except Exception: pass
ADDR = re.compile(r'0x[0-9a-fA-F]{40}|[1-9A-HJ-NP-Za-km-z]{32,44}')
FARM = re.compile(r"qualif|allocation|eligib|airdrop|claim|reward", re.I)
def same(p, e):
    tx = p.get("text") or ""; a = e["address"].lower(); nm = (e["name"] or "").lower(); sym = "$" + (e["symbol"] or "").lower()
    if a in tx.lower(): return True
    if sym not in tx.lower(): return False
    return not [x for x in ADDR.findall(tx) if x.lower() != a]
def human(p):
    tx = re.sub(r'@\w+|\$\w+|https?://\S+', '', p.get("text") or "").strip()
    if p.get("isReply") and len(tx.split()) <= 4: return False
    if FARM.search(p.get("text") or "") and "http" in (p.get("text") or ""): return False
    return True
rows = []
for k, r in X.items():
    e = EV.get(k)
    if not e or e["label"] not in ("SUCCESS", "FAKE"): continue
    T = e["t"]
    L = [p for p in r["last10"]["real"] if same(p, e) and human(p)]
    B = [p for p in r["base"]["real"] if same(p, e) and human(p)]
    cmp = lambda P: [p for p in P if (p.get("likeCount") or 0) >= 3]
    hrs = [sum(1 for p in L if T - (h + 1) * 3600 <= p["ts"] < T - h * 3600) for h in range(9, -1, -1)]
    la = {p["user"] for p in L}; ba = {p["user"] for p in B}
    eps = 0.5
    d = dict(k=k, coin=e["address"], chain=e["chain"], t=T, y=1 if e["label"] == "SUCCESS" else 0,
             complete=r["last10"]["complete"] and r["base"]["complete"],
             n10=len(L), nb=len(B), c10=len(cmp(L)), cb=len(cmp(B)),
             rate_ratio=(len(L) / 10 + 0.05) / (len(B) / 24 + 0.05),
             cmp_ratio=(len(cmp(L)) / 10 + 0.05) / (len(cmp(B)) / 24 + 0.05),
             authors10=len(la), new_auth_share=(len(la - ba) / len(la)) if la else None,
             last3_share=(sum(hrs[-3:]) / len(L)) if L else None,
             likes10=sum(p.get("likeCount") or 0 for p in L), likes_per_post=(st.mean([p.get("likeCount") or 0 for p in L]) if L else None),
             big10=sum(1 for p in L if (p.get("fol") or 0) >= 10000), big_share=(sum(1 for p in L if (p.get("fol") or 0) >= 10000) / len(L)) if L else None,
             reply_share=(sum(1 for p in L if p.get("isReply")) / len(L)) if L else None,
             spam10=r["last10"]["spam"], silent10=1 if not L else 0,
             vol_ratio=(e["vol_last10"] / 10 + 1e-9) / (e["vol_base24"] / 24 + 1e-9), ret_last10=e["ret_last10"],
             pullback=e["pullback"], hours_since_ath=e["hours_since_ath"], age_h=e["age_h"], mc=e["mc_old_ath"] or 0)
    rows.append(d)
json.dump(rows, open("/opt/bostudy/rows_tag.json", "w"))
print("scored events with X:", len(rows), "success", sum(r["y"] for r in rows), "coins", len({r["coin"] for r in rows}))
base = st.mean(r["y"] for r in rows); print(f"base success rate {base:.0%}")
med = lambda v: st.median(v) if v else float("nan")
print("\nMedians, SUCCESS vs FAKE")
F = ["n10", "nb", "rate_ratio", "c10", "cmp_ratio", "authors10", "new_auth_share", "last3_share", "likes_per_post", "big10", "big_share", "reply_share", "silent10", "vol_ratio", "ret_last10", "pullback", "hours_since_ath", "age_h", "mc"]
for f in F:
    s = [r[f] for r in rows if r["y"] == 1 and r[f] is not None]; fk = [r[f] for r in rows if r["y"] == 0 and r[f] is not None]
    print(f"  {f:16s} success {med(s):10.3g}   fake {med(fk):10.3g}   (n {len(s)}/{len(fk)})")
def boot(sub_hi, sub_lo, n=1000):
    coins = list({r["coin"] for r in sub_hi + sub_lo}); by = collections.defaultdict(list)
    for r in sub_hi: by[r["coin"]].append((1, r["y"]))
    for r in sub_lo: by[r["coin"]].append((0, r["y"]))
    out = []
    for _ in range(n):
        h = l = hn = ln = 0
        for c in random.choices(coins, k=len(coins)):
            for g, y in by[c]:
                if g: h += y; hn += 1
                else: l += y; ln += 1
        if hn and ln: out.append(h / hn - l / ln)
    out.sort(); return out[int(.05 * len(out))], out[int(.95 * len(out))]
print("\nTop third vs bottom third: success rate (90% CI coin bootstrap) | halves: early/late time, RH/SOL")
random.seed(1)
ts = sorted(r["t"] for r in rows); tmid = ts[len(ts) // 2]
for f in F:
    v = [r for r in rows if r[f] is not None]
    if len(v) < 60: continue
    xsv = sorted(r[f] for r in v); lo, hi = xsv[len(xsv) // 3], xsv[2 * len(xsv) // 3]
    if lo == hi: H = [r for r in v if r[f] > lo]; Lo = [r for r in v if r[f] <= lo]
    else: H = [r for r in v if r[f] >= hi]; Lo = [r for r in v if r[f] <= lo]
    if len(H) < 15 or len(Lo) < 15: continue
    a = st.mean(r["y"] for r in H); b = st.mean(r["y"] for r in Lo); ci = boot(H, Lo)
    def sp(cond):
        h = [r["y"] for r in H if cond(r)]; l = [r["y"] for r in Lo if cond(r)]
        return f"{(st.mean(h) - st.mean(l)) * 100:+.0f}" if h and l else "na"
    print(f"  {f:16s} high {a:4.0%} (n{len(H)}) low {b:4.0%} (n{len(Lo)}) diff {(a-b)*100:+.0f} pts CI [{ci[0]*100:+.0f},{ci[1]*100:+.0f}] | early {sp(lambda r: r['t'] < tmid)} late {sp(lambda r: r['t'] >= tmid)} RH {sp(lambda r: r['chain'] == 'robinhood')} SOL {sp(lambda r: r['chain'] == 'sol')}")