import json, urllib.request, urllib.parse, time, os, pandas as pd
from concurrent.futures import ThreadPoolExecutor
KEY=open('/tmp/run/.xk').read().strip()
d=pd.read_pickle('clean.pkl')
x=d[(d.start=='small')&(d['ts_500000'].notna())]
OUT='xdata2.json'; out=json.load(open(OUT)) if os.path.exists(OUT) else {}
def q(query,pages=2):
    tw=[];cur=''
    for p in range(pages):
        u='https://api.twitterapi.io/twitter/tweet/advanced_search?'+urllib.parse.urlencode(dict(query=query,queryType='Latest',cursor=cur))
        dd=None
        for i in range(4):
            try: dd=json.loads(urllib.request.urlopen(urllib.request.Request(u,headers={'X-API-Key':KEY,'User-Agent':'Mozilla/5.0'}),timeout=30).read()); break
            except Exception: time.sleep(3+3*i)
        if not dd: break
        tw+=dd.get('tweets') or []
        if not dd.get('has_next_page'): break
        cur=dd.get('next_cursor') or ''
    return [dict(t=z.get('createdAt'),a=(z.get('author') or {}).get('userName'),f=(z.get('author') or {}).get('followers')) for z in tw]
def one(r):
    if r.tok in out: return
    T1=int(r.t0+r['ts_500000'])
    out[r.tok]=dict(before=q('%s since_time:%d until_time:%d'%(r.tok,int(r.t0)-3600,T1)),after1h=q('%s since_time:%d until_time:%d'%(r.tok,T1,T1+3600)))
rows=[r for _,r in x.iterrows()]
with ThreadPoolExecutor(4) as ex: list(ex.map(one,rows))
json.dump(out,open(OUT,'w')); print('done',len(out))