#!/usr/bin/env python3
# ROT A1: point-in-time CMC listings (survivor free), daily 2019-01-01..yesterday, top 1500, trimmed gz.
import json, gzip, os, sys, time, datetime as dt, urllib.request, threading
OUT='/opt/rot/analyst/raw/cmc'; os.makedirs(OUT, exist_ok=True)
UA='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/127 Safari/537.36'
URL='https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listings/historical?date={d}&limit=1500&start=1&convertId=2781'
import os as _o
start=dt.date.fromisoformat(_o.environ.get("CMC_START","2019-01-01")); end=dt.date.fromisoformat(_o.environ.get("CMC_END",str(dt.date.today()-dt.timedelta(days=1))))
days=[start+dt.timedelta(i) for i in range((end-start).days+1)]
todo=[d for d in days if not os.path.exists(f'{OUT}/{d}.json.gz')]
print('todo',len(todo),flush=True)
lock=threading.Lock(); fails=[]
def get(d):
    req=urllib.request.Request(URL.format(d=d),headers={'User-Agent':UA,'Accept':'application/json'})
    with urllib.request.urlopen(req,timeout=60) as r: return json.load(r)
def work(chunk):
    for d in chunk:
        for attempt in (1,2):
            try:
                j=get(d); L=j['data']; break
            except Exception as e:
                L=None
                if attempt==1: time.sleep(30)
                else:
                    with lock: fails.append((str(d),str(e)[:80]))
        if L:
            rows=[[x['id'],x['symbol'],x['name'],x['slug'],x['cmcRank'],x['quotes'][0]['price'],x['quotes'][0]['volume24h'],x['quotes'][0]['marketCap'],x.get('circulatingSupply'),x.get('dateAdded'),x.get('tags',[])] for x in L]
            tmp=f'{OUT}/{d}.tmp'
            with gzip.open(tmp,'wt') as f: json.dump(rows,f)
            os.replace(tmp,f'{OUT}/{d}.json.gz')
        time.sleep(0.6)
N=2; th=[threading.Thread(target=work,args=(todo[i::N],)) for i in range(N)]
[t.start() for t in th]; [t.join() for t in th]
print('done fails',len(fails),fails[:20],flush=True)