import json, urllib.request, urllib.parse, time, sys, os
from concurrent.futures import ThreadPoolExecutor
KEY = open('/tmp/run/.xk').read().strip()
rows = json.load(open('/tmp/run/ladder.json'))
out = json.load(open('/tmp/run/xdata.json')) if os.path.exists('/tmp/run/xdata.json') else {}
def q(query, pages=3):
    tw = []; cur = ''
    for p in range(pages):
        u = 'https://api.twitterapi.io/twitter/tweet/advanced_search?' + urllib.parse.urlencode(dict(query=query, queryType='Latest', cursor=cur))
        for i in range(4):
            try:
                d = json.loads(urllib.request.urlopen(urllib.request.Request(u, headers={'X-API-Key': KEY, 'User-Agent': 'Mozilla/5.0'}), timeout=30).read()); break
            except Exception as e: d = None; time.sleep(3 + 3 * i)
        if not d: break
        tw += d.get('tweets') or []
        if not d.get('has_next_page'): break
        cur = d.get('next_cursor') or ''
    return [dict(t=x.get('createdAt'), a=(x.get('author') or {}).get('userName'), f=(x.get('author') or {}).get('followers'), v=x.get('viewCount'), l=x.get('likeCount')) for x in tw]
def one(r):
    if r['tok'] in out: return
    T1 = int(r['T1'])
    out[r['tok']] = dict(before=q('%s since_time:%d until_time:%d' % (r['tok'], T1 - 86400, T1)), after6=q('%s since_time:%d until_time:%d' % (r['tok'], T1, T1 + 6 * 3600), pages=2))
with ThreadPoolExecutor(4) as ex: list(ex.map(one, rows))
json.dump(out, open('/tmp/run/xdata.json', 'w')); print('done', len(out))