#!/usr/bin/env python3
"""ROT A2 flow feed fetcher (research copy). Coder: wrap as the daily loader.
Writes tidy rows: date(UTC day), source, metric, entity, value -> data/flows_daily.csv.gz
Every fetch: browser UA, no retry loops, one call per item, per-run cap."""
import json, time, gzip, csv, os, sys, urllib.request, urllib.error, urllib.parse
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
os.makedirs(OUT, exist_ok=True)
CAP = 700; calls = 0; errors = []

def get(url, data=None):
    global calls
    if calls >= CAP: raise RuntimeError("per-run cap")
    calls += 1
    req = urllib.request.Request(url, data=data, headers={"User-Agent": UA, "Content-Type": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=40) as r: return json.loads(r.read())
    except Exception as e:
        errors.append(f"{url[:120]} {e}"); return None
    finally: time.sleep(0.35)

def day(ts): return time.strftime("%Y-%m-%d", time.gmtime(int(float(ts))))
rows = []
def add(d, src, metric, ent, v):
    if v is None: return
    rows.append((d, src, metric, ent, float(v)))

CHAINS = ["Ethereum","Solana","Base","BSC","Arbitrum","Tron","Hyperliquid L1","Robinhood Chain","Sui","Avalanche",
          "Sonic","TON","Polygon","Monad","Ink","Plasma","Aptos","OP Mainnet","Abstract","Unichain"]

# 1. stablecoin supply per chain (+ total)
sc = get("https://stablecoins.llama.fi/stablecoinchains") or []
scnames = {c["name"] for c in sc}
for ch in CHAINS + ["all"]:
    if ch != "all" and ch not in scnames: errors.append(f"stable: no chain {ch}"); continue
    j = get("https://stablecoins.llama.fi/stablecoincharts/" + urllib.parse.quote(ch))
    for p in j or []:
        add(day(p["date"]), "llama_stable", "stable_supply_usd", ch, (p.get("totalCirculatingUSD") or {}).get("peggedUSD"))

# 2. DEX volume, fees, TVL per chain
for ch in CHAINS:
    q = urllib.parse.quote(ch)
    j = get(f"https://api.llama.fi/overview/dexs/{q}?excludeTotalDataChart=false&excludeTotalDataChartBreakdown=true")
    for t, v in (j or {}).get("totalDataChart", []): add(day(t), "llama_dex", "dex_volume_usd", ch, v)
    j = get(f"https://api.llama.fi/overview/fees/{q}?excludeTotalDataChart=false&excludeTotalDataChartBreakdown=true&dataType=dailyFees")
    for t, v in (j or {}).get("totalDataChart", []): add(day(t), "llama_fees", "fees_usd", ch, v)
    j = get(f"https://api.llama.fi/v2/historicalChainTvl/{q}")
    for p in j or []: add(day(p["date"]), "llama_tvl", "tvl_usd", ch, p["tvl"])

# 3. launchpads + retail trading apps (fees, split by chain)
PADS = ["pump.fun","pumpswap","bonk.fun-launchpad","launchlab","meteora-dynamic-bonding-curve","bags","four.meme",
        "flap-sh","clanker","zora-coins","virtuals-protocol","pons-v2","pons-v1","fables","o1-launchpad","pools",
        "flaunch","liquidlaunch","moonshot-create","believe","axiom","gmgn","fomo-wallet","photon","bullx","maestro","bonkbot"]
for slug in PADS:
    j = get(f"https://api.llama.fi/summary/fees/{slug}?dataType=dailyFees")
    if not j: continue
    br = j.get("totalDataChartBreakdown") or []
    if br:
        for t, d in br:
            for chn, v in d.items():
                tot = sum(x for x in v.values()) if isinstance(v, dict) else v
                add(day(t), "llama_pad", "pad_fees_usd", f"{slug}|{chn}", tot)
    else:
        for t, v in j.get("totalDataChart", []): add(day(t), "llama_pad", "pad_fees_usd", f"{slug}|all", v)

# 4. CEX: Binance funding (8h -> daily sum) and Bybit OI daily
MAJ = ["BTC","ETH","SOL","BNB","HYPE","SUI","AVAX","ARB","TRX","TON","APT","DOGE","XRP","S"]
for s in MAJ:
    sym = s + "USDT"; start = int(time.mktime((2020,1,1,0,0,0,0,0,0))*1000); fund = {}
    for _ in range(8):
        j = get(f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={sym}&startTime={start}&limit=1000")
        if not j: break
        for p in j: fund[day(p["fundingTime"]/1000)] = fund.get(day(p["fundingTime"]/1000), 0) + float(p["fundingRate"])
        if len(j) < 1000: break
        start = j[-1]["fundingTime"] + 1
    for d, v in fund.items(): add(d, "binance", "funding_daily", s, v)
    cur = ""
    for _ in range(12):
        j = get(f"https://api.bybit.com/v5/market/open-interest?category=linear&symbol={sym}&intervalTime=1d&limit=200" + (f"&cursor={cur}" if cur else ""))
        res = (j or {}).get("result") or {}
        for p in res.get("list", []): add(day(int(p["timestamp"])/1000), "bybit", "oi_contracts", s, p["openInterest"])
        cur = res.get("nextPageCursor") or ""
        if not cur or not res.get("list"): break

# 5. Prices: Binance spot daily closes (USDT) for chain natives + BTC
NAT = ["BTC","ETH","SOL","BNB","ARB","TRX","SUI","AVAX","S","TON","POL","APT","OP","HYPE","DOGE","XRP"]
for s in NAT:
    start = int(time.mktime((2019,1,1,0,0,0,0,0,0))*1000)
    for _ in range(4):
        j = get(f"https://api.binance.com/api/v3/klines?symbol={s}USDT&interval=1d&startTime={start}&limit=1000")
        if not j: break
        for k in j:
            add(day(k[0]/1000), "binance", "close_usd", s, k[4]); add(day(k[0]/1000), "binance", "quote_vol_usd", s, k[7])
        if len(j) < 1000: break
        start = j[-1][0] + 86400000

with gzip.open(os.path.join(OUT, "flows_daily.csv.gz"), "wt", newline="") as f:
    w = csv.writer(f); w.writerow(["date","source","metric","entity","value"]); w.writerows(rows)
json.dump({"rows": len(rows), "calls": calls, "errors": errors, "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())},
          open(os.path.join(OUT, "fetch_meta.json"), "w"), indent=1)
print(len(rows), "rows", calls, "calls", len(errors), "errors")