#!/usr/bin/env python3
"""ROT A2 fetch v2 (7 Oct 2026): funding on other venues, non-dollar counts, attention.
Appends tidy rows (date, source, metric, entity, value) to data/flows2_daily.csv.gz and hourly HL funding to data/hl_funding_1h.csv.gz.
Browser UA, no retry loops, per-run cap."""
import json, time, gzip, csv, os, urllib.request, urllib.parse
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
CAP = 2500; calls = 0; errors = []
def get(url, body=None, gap=0.25):
    global calls
    if calls >= CAP: raise RuntimeError("cap")
    calls += 1
    h = {"User-Agent": UA, "Content-Type": "application/json", "Api-User-Agent": "rot-a2 research (benkorsa)"}
    req = urllib.request.Request(url, data=json.dumps(body).encode() if body else None, headers=h)
    try:
        with urllib.request.urlopen(req, timeout=40) as r: return json.loads(r.read())
    except Exception as e:
        errors.append(f"{url[:110]} {e}"); return None
    finally: time.sleep(gap)
day = lambda ms: time.strftime("%Y-%m-%d", time.gmtime(int(ms) / 1000))
rows = []
def add(d, s, m, e, v):
    if v is not None: rows.append((d, s, m, e, float(v)))
MAJ = ["BTC","ETH","SOL","BNB","HYPE","SUI","AVAX","ARB","TRX","TON","APT","DOGE","XRP","S","OP","POL"]
T0 = int(time.mktime((2020,1,1,0,0,0,0,0,0)) * 1000); NOW = int(time.time() * 1000)

# 1. Bybit funding (8h), paged backwards by endTime
for s in MAJ:
    end, f = NOW, {}
    for _ in range(45):
        j = get(f"https://api.bybit.com/v5/market/funding/history?category=linear&symbol={s}USDT&endTime={end}&limit=200")
        L = ((j or {}).get("result") or {}).get("list") or []
        for p in L: d = day(p["fundingRateTimestamp"]); f[d] = f.get(d, 0) + float(p["fundingRate"])
        if len(L) < 200: break
        end = int(L[-1]["fundingRateTimestamp"]) - 1
        if end < T0: break
    for d, v in f.items(): add(d, "bybit", "funding_daily", s, v)

# 2. OKX funding (only ~3 months kept by OKX)
for s in MAJ:
    after, f = "", {}
    for _ in range(4):
        j = get(f"https://www.okx.com/api/v5/public/funding-rate-history?instId={s}-USDT-SWAP&limit=100" + (f"&after={after}" if after else ""))
        L = (j or {}).get("data") or []
        for p in L: d = day(p["fundingTime"]); f[d] = f.get(d, 0) + float(p["realizedRate"] or p["fundingRate"])
        if len(L) < 100: break
        after = L[-1]["fundingTime"]
    for d, v in f.items(): add(d, "okx", "funding_daily", s, v)

# 3. Hyperliquid funding (hourly) since May 2023
hl = []
HLC = {"POL": "MATIC"}
for s in MAJ:
    c = HLC.get(s, s); start = int(time.mktime((2023,5,1,0,0,0,0,0,0)) * 1000)
    for _ in range(80):
        j = get("https://api.hyperliquid.xyz/info", {"type": "fundingHistory", "coin": c, "startTime": start}, gap=0.3)
        if not j: break
        for p in j: hl.append((int(p["time"]), s, float(p["fundingRate"]), float(p["premium"])))
        if len(j) < 500: break
        start = int(j[-1]["time"]) + 1
with gzip.open(f"{OUT}/hl_funding_1h.csv.gz", "wt", newline="") as fh:
    w = csv.writer(fh); w.writerow(["ts_ms", "coin", "funding", "premium"]); w.writerows(hl)
hd = {}
for t, s, f, _ in hl: k = (day(t), s); hd[k] = hd.get(k, 0) + f
for (d, s), v in hd.items(): add(d, "hyperliquid", "funding_daily", s, v)

# 4. Artemis counts (not dollars): DAU, NEW_USERS, TXNS per chain
AIDS = ["ethereum","solana","bsc","arbitrum","tron","sui","avalanche","sonic","ton","polygon","aptos","optimism","hyperliquid","base"]
for m in ["DAU", "NEW_USERS", "TXNS"]:
    for i in range(0, len(AIDS), 5):
        ids = ",".join(AIDS[i:i+5])
        j = get(f"https://api.artemisxyz.com/data/{m}/?artemisIds={ids}&startDate=2019-01-01&endDate=" + time.strftime("%Y-%m-%d"), gap=1.0)
        for a, v in (((j or {}).get("data") or {}).get("artemis_ids") or {}).items():
            if isinstance(v.get(m), list):
                for p in v[m]: add(p["date"], "artemis", m.lower(), a, p["val"])
            else: errors.append(f"artemis {m} {a}: {v.get(m)}")

# 5. Wikipedia pageviews (attention, not dollars)
WIKI = {"Bitcoin": "Bitcoin", "Ethereum": "Ethereum", "Solana": "Solana_(blockchain_platform)", "BNB": "Binance",
        "Tron": "Tron_(cryptocurrency)", "TON": "TON_(blockchain)", "Avalanche": "Avalanche_(blockchain_platform)",
        "Polygon": "Polygon_(blockchain)", "Cardano": "Cardano_(blockchain_platform)", "Dogecoin": "Dogecoin",
        "XRP": "Ripple_Labs", "Memecoin": "Meme_coin", "Cryptocurrency": "Cryptocurrency", "Stablecoin": "Stablecoin",
        "NFT": "Non-fungible_token", "DeFi": "Decentralized_finance", "Hyperliquid": "Hyperliquid", "Sui": "Sui_(blockchain)"}
for k, art in WIKI.items():
    j = get("https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/user/"
            + urllib.parse.quote(art, safe="") + "/daily/20150701/" + time.strftime("%Y%m%d"), gap=1.5)
    for p in (j or {}).get("items", []):
        ts = p["timestamp"]; add(f"{ts[:4]}-{ts[4:6]}-{ts[6:8]}", "wikipedia", "pageviews", k, p["views"])

with gzip.open(f"{OUT}/flows2_daily.csv.gz", "wt", newline="") as fh:
    w = csv.writer(fh); w.writerow(["date", "source", "metric", "entity", "value"]); w.writerows(rows)
json.dump({"rows": len(rows), "hl_rows": len(hl), "calls": calls, "errors": errors[:60], "n_err": len(errors),
           "utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, open(f"{OUT}/fetch2_meta.json", "w"), indent=1)
print(len(rows), "rows", len(hl), "hl", calls, "calls", len(errors), "errors")