import pandas as pd, numpy as np, json, sys, hashlib, threading
sys.path.insert(0,'/tmp/run'); from rpc import rpc
df=pd.read_pickle('df.pkl').sort_values('t0').drop_duplicates('tok',keep='first')
old=df.rt_created.notna()&(df.rt_created<df.t0-86400)
d=df[~old&(df.mc_first<5e6)].copy()
d['pk']=d.pk_sell.fillna(0)
d['hook_n']=d.hook.map(d.hook.value_counts())
C=json.load(open('codes.json'))
todo=[h for h in set(d.hook.dropna())-{'0x'+'0'*40} if h not in C]
print('new hooks',len(todo))
lock=threading.Lock()
def w():
    while True:
        with lock:
            if not todo: return
            a=todo.pop()
        c=rpc('eth_getCode',[a,'latest']); c=c if isinstance(c,str) else ''
        with lock: C[a]=(len(c)//2-1, hashlib.md5(c.encode()).hexdigest()[:8], None)
ts=[threading.Thread(target=w) for _ in range(3)]; [t.start() for t in ts]; [t.join() for t in ts]
json.dump(C,open('codes.json','w'))
d['hfam']=d.hook.map(lambda h: 'none' if h=='0x'+'0'*40 else ('%d_%s'%(C[h][0],C[h][1]) if h in C else '?'))
d['fam']=np.where(d.hfam=='5059_7adb2ec7','campaign_weth',np.where(d.hfam=='341_6de2bb44','bundle_eth','organic'))
d['start']=pd.cut(d.mc_first,[0,2e5,1e6,5e6],labels=['small','mid','big'])
d.to_pickle('clean.pkl')
print(len(d)); print(d.groupby(['start','fam'],observed=True).size())
for L in [5e5,1e6,2e6,5e6,1e7,2e7]: print('sellable6h >=%g'%L, (d.pk>=L).sum(), ' organic', ((d.pk>=L)&(d.fam=='organic')).sum())
print('rt7d>=5M',(d.pk_rt.fillna(0)>=5e6).sum(),' >=20M',(d.pk_rt.fillna(0)>=2e7).sum())