import json, gzip, urllib.request, urllib.parse, time, os
from concurrent.futures import ThreadPoolExecutor
KEY=open('/tmp/run/.xk').read().strip()
L=json.load(open('list.json')); R=json.load(gzip.open('/tmp/run/rt.json.gz','rt'))
K={k.lower():v for k,v in R['kline'].items()}
OUT='xstage.json'; out=json.load(open(OUT)) if os.path.exists(OUT) else {}
def q(query,pages=2):
    tw=[];cur=''
    for p in range(pages):
        u='https://api.twitterapi.io/twitter/tweet/advanced_search?'+urllib.parse.urlencode(dict(query=query,queryType='Latest',cursor=cur))
        dd=None
        for i in range(4):
            try: dd=json.loads(urllib.request.urlopen(urllib.request.Request(u,headers={'X-API-Key':KEY,'User-Agent':'Mozilla/5.0'}),timeout=30).read()); break
            except Exception: time.sleep(3+3*i)
        if not dd: break
        tw+=dd.get('tweets') or []
        if not dd.get('has_next_page'): break
        cur=dd.get('next_cursor') or ''
    return [dict(t=z.get('createdAt'),a=(z.get('author') or {}).get('userName'),f=(z.get('author') or {}).get('followers'),v=z.get('viewCount')) for z in tw]
W=[('pre24h',-86400,0),('h0_1',0,3600),('h1_4',3600,4*3600),('h4_24',4*3600,86400),('d1_3',86400,3*86400)]
def one(it):
    t=it['tok']
    if t in out: return
    k=K.get(t)
    if not k: return
    T0=next((c[0] for c in k if c[5]>=5000),None)
    if T0 is None: return
    out[t]=dict(T0=T0,**{nm:q('%s since_time:%d until_time:%d'%(t,T0+a,T0+b)) for nm,a,b in W})
with ThreadPoolExecutor(4) as ex: list(ex.map(one,L['cand']+L['ctl']))
json.dump(out,open(OUT,'w')); print('done',len(out))