dnd-srd-api/scripts/fetch_data.py

#!/usr/bin/env python3
"""
Fetch all SRD 5.2 data from Open5e API and cache as flat JSON files.
"""

import json
import os
import sys
import time
import urllib.request
import urllib.error

DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data")
SRD_FILTER = "document__key__in=srd-2024"

SOURCES = {
    "spells": f"https://api.open5e.com/v2/spells/?limit=100&{SRD_FILTER}",
    "creatures": f"https://api.open5e.com/v2/creatures/?limit=100&{SRD_FILTER}",
    "classes": f"https://api.open5e.com/v2/classes/?limit=100&{SRD_FILTER}",
    "magic-items": "https://api.open5e.com/v2/magicitems/?limit=100",
    "equipment": f"https://api.open5e.com/v2/items/?limit=100&{SRD_FILTER}",
}

os.makedirs(DATA_DIR, exist_ok=True)


def fetch_page(url: str, retries: int = 3) -> dict:
    """Fetch a single page from the API with retries."""
    for attempt in range(retries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": "Cupcake-DnD-API/1.0"})
            with urllib.request.urlopen(req, timeout=30) as resp:
                data = json.loads(resp.read().decode())
            return data
        except Exception as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  ⚠ Retry {attempt + 1}/{retries} after {wait}s: {e}")
                time.sleep(wait)
            else:
                raise


def fetch_all(name: str, first_url: str) -> list:
    """Fetch all pages of a resource, returning combined results."""
    print(f"\n📦 Fetching {name}...")
    all_results = []
    url = first_url
    page = 0

    while url:
        page += 1
        print(f"  📄 Page {page}: {url[:80]}...")
        data = fetch_page(url)
        results = data.get("results", [])
        all_results.extend(results)
        print(f"     Got {len(results)} items (total: {len(all_results)})")
        url = data.get("next")

    print(f"✅ {name}: {len(all_results)} total items")
    return all_results


def simplify_spell(s: dict) -> dict:
    """Extract key fields from a spell."""
    return {
        "key": s.get("key", "").replace("srd-2024_", ""),
        "name": s.get("name", ""),
        "level": s.get("level", 0),
        "school": s.get("school", {}).get("name") if isinstance(s.get("school"), dict) else s.get("school"),
        "casting_time": s.get("casting_time", ""),
        "range": s.get("range", ""),
        "components": {"material": s.get("material"), "verbal": s.get("verbal"), "somatic": s.get("somatic")},
        "duration": s.get("duration", ""),
        "concentration": s.get("concentration", False),
        "ritual": s.get("ritual", False),
        "description": s.get("desc", ""),
        "higher_level": s.get("higher_level", ""),
        "classes": [c.get("name") for c in (s.get("classes") or [])] if isinstance(s.get("classes"), list) else [],
        "subclasses": [c.get("name") for c in (s.get("subclasses") or [])] if isinstance(s.get("subclasses"), list) else [],
    }


def simplify_creature(c: dict) -> dict:
    """Extract key fields from a creature."""
    return {
        "key": c.get("key", ""),
        "name": c.get("name", ""),
        "size": c.get("size", {}).get("name") if isinstance(c.get("size"), dict) else c.get("size"),
        "type": c.get("type", {}).get("name") if isinstance(c.get("type"), dict) else c.get("type"),
        "alignment": c.get("alignment", ""),
        "armor_class": c.get("armor_class", 0),
        "hit_points": c.get("hit_points", 0),
        "hit_dice": c.get("hit_dice", ""),
        "speed": c.get("speed", {}),
        "strength": c.get("strength", 10),
        "dexterity": c.get("dexterity", 10),
        "constitution": c.get("constitution", 10),
        "intelligence": c.get("intelligence", 10),
        "wisdom": c.get("wisdom", 10),
        "charisma": c.get("charisma", 10),
        "saving_throws": c.get("saving_throws", {}),
        "skills": c.get("skills", {}),
        "damage_vulnerabilities": c.get("damage_vulnerabilities", []),
        "damage_resistances": c.get("damage_resistances", []),
        "damage_immunities": c.get("damage_immunities", []),
        "condition_immunities": c.get("condition_immunities", []),
        "senses": c.get("senses", ""),
        "languages": c.get("languages", ""),
        "challenge_rating": c.get("challenge_rating", "0"),
        "xp": c.get("xp", 0),
        "traits": [{"name": t.get("name"), "desc": t.get("desc")} for t in (c.get("traits") or [])],
        "actions": [{"name": a.get("name"), "desc": a.get("desc")} for a in (c.get("actions") or [])],
        "legendary_actions": [{"name": a.get("name"), "desc": a.get("desc")} for a in (c.get("legendary_actions") or [])],
        "description": c.get("desc", ""),
    }


def simplify_class(c: dict) -> dict:
    """Extract key fields from a class."""
    return {
        "key": c.get("key", ""),
        "name": c.get("name", ""),
        "hit_dice": c.get("hit_dice", ""),
        "description": c.get("desc", ""),
        "spellcasting_ability": c.get("spellcasting_ability", {}).get("key") if isinstance(c.get("spellcasting_ability"), dict) else c.get("spellcasting_ability"),
        "subclasses": [s.get("name") for s in (c.get("subclasses") or [])] if isinstance(c.get("subclasses"), list) else [],
    }


def simplify_magic_item(m: dict) -> dict:
    """Extract key fields from a magic item."""
    return {
        "key": m.get("key", ""),
        "name": m.get("name", ""),
        "type": m.get("type", {}).get("name") if isinstance(m.get("type"), dict) else m.get("type"),
        "rarity": m.get("rarity", {}).get("name") if isinstance(m.get("rarity"), dict) else m.get("rarity"),
        "requires_attunement": m.get("requires_attunement", ""),
        "description": m.get("desc", ""),
    }


def simplify_item(i: dict) -> dict:
    """Extract key fields from equipment/item."""
    return {
        "key": i.get("key", ""),
        "name": i.get("name", ""),
        "type": i.get("type", {}).get("name") if isinstance(i.get("type"), dict) else i.get("type"),
        "cost": i.get("cost", ""),
        "weight": i.get("weight", ""),
        "description": i.get("desc", ""),
        "armor_class": i.get("armor_class"),
        "stealth_disadvantage": i.get("stealth_disadvantage", False),
        "strength_requirement": i.get("strength_requirement"),
        "damage": i.get("damage", {}).get("damage_dice") if isinstance(i.get("damage"), dict) else None,
        "properties": [p.get("name") for p in (i.get("properties") or [])] if isinstance(i.get("properties"), list) else [],
    }


SIMPLIFIERS = {
    "spells": simplify_spell,
    "creatures": simplify_creature,
    "classes": simplify_class,
    "magic-items": simplify_magic_item,
    "equipment": simplify_item,
}


def main():
    for name, first_url in SOURCES.items():
        try:
            raw_data = fetch_all(name, first_url)
            simplifier = SIMPLIFIERS.get(name)
            if simplifier:
                simplified = [simplifier(item) for item in raw_data]
            else:
                simplified = raw_data

            # Build index: name -> slug and key -> item
            indexed = {
                item.get("key"): item
                for item in simplified
                if item.get("key")
            }

            # Also build a name index
            name_index = {}
            for item in simplified:
                name_lower = item.get("name", "").lower()
                slug = item.get("key", "")
                if name_lower:
                    name_index[name_lower] = slug

            output = {
                "count": len(simplified),
                "results": simplified,
                "_index": {
                    "by_key": {k: True for k in indexed.keys()},
                    "by_name": name_index,
                }
            }

            # Remove _index for display but keep for reference
            filepath = os.path.join(DATA_DIR, f"{name}.json")
            with open(filepath, "w") as f:
                json.dump(output, f, indent=2)

            print(f"💾 Saved {len(simplified)} items to {filepath}")

            # Also save raw data for debugging
            raw_filepath = os.path.join(DATA_DIR, f"{name}_raw.json")
            with open(raw_filepath, "w") as f:
                json.dump({"count": len(raw_data), "results": raw_data}, f, indent=2)
            print(f"💾 Raw backup saved to {raw_filepath}")

        except Exception as e:
            print(f"❌ Failed to fetch {name}: {e}")
            sys.exit(1)

    print("\n✨ All data fetched successfully!")
    print(json.dumps({"status": "ok", "dir": DATA_DIR}, indent=2))


if __name__ == "__main__":
    main()