226 lines
No EOL
8.6 KiB
Python
226 lines
No EOL
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Fetch all SRD 5.2 data from Open5e API and cache as flat JSON files.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
import urllib.error
|
|
|
|
DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data")
|
|
SRD_FILTER = "document__key__in=srd-2024"
|
|
|
|
SOURCES = {
|
|
"spells": f"https://api.open5e.com/v2/spells/?limit=100&{SRD_FILTER}",
|
|
"creatures": f"https://api.open5e.com/v2/creatures/?limit=100&{SRD_FILTER}",
|
|
"classes": f"https://api.open5e.com/v2/classes/?limit=100&{SRD_FILTER}",
|
|
"magic-items": "https://api.open5e.com/v2/magicitems/?limit=100",
|
|
"equipment": f"https://api.open5e.com/v2/items/?limit=100&{SRD_FILTER}",
|
|
}
|
|
|
|
os.makedirs(DATA_DIR, exist_ok=True)
|
|
|
|
|
|
def fetch_page(url: str, retries: int = 3) -> dict:
|
|
"""Fetch a single page from the API with retries."""
|
|
for attempt in range(retries):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "Cupcake-DnD-API/1.0"})
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
return data
|
|
except Exception as e:
|
|
if attempt < retries - 1:
|
|
wait = 2 ** attempt
|
|
print(f" ⚠ Retry {attempt + 1}/{retries} after {wait}s: {e}")
|
|
time.sleep(wait)
|
|
else:
|
|
raise
|
|
|
|
|
|
def fetch_all(name: str, first_url: str) -> list:
|
|
"""Fetch all pages of a resource, returning combined results."""
|
|
print(f"\n📦 Fetching {name}...")
|
|
all_results = []
|
|
url = first_url
|
|
page = 0
|
|
|
|
while url:
|
|
page += 1
|
|
print(f" 📄 Page {page}: {url[:80]}...")
|
|
data = fetch_page(url)
|
|
results = data.get("results", [])
|
|
all_results.extend(results)
|
|
print(f" Got {len(results)} items (total: {len(all_results)})")
|
|
url = data.get("next")
|
|
|
|
print(f"✅ {name}: {len(all_results)} total items")
|
|
return all_results
|
|
|
|
|
|
def simplify_spell(s: dict) -> dict:
|
|
"""Extract key fields from a spell."""
|
|
return {
|
|
"key": s.get("key", "").replace("srd-2024_", ""),
|
|
"name": s.get("name", ""),
|
|
"level": s.get("level", 0),
|
|
"school": s.get("school", {}).get("name") if isinstance(s.get("school"), dict) else s.get("school"),
|
|
"casting_time": s.get("casting_time", ""),
|
|
"range": s.get("range", ""),
|
|
"components": {"material": s.get("material"), "verbal": s.get("verbal"), "somatic": s.get("somatic")},
|
|
"duration": s.get("duration", ""),
|
|
"concentration": s.get("concentration", False),
|
|
"ritual": s.get("ritual", False),
|
|
"description": s.get("desc", ""),
|
|
"higher_level": s.get("higher_level", ""),
|
|
"classes": [c.get("name") for c in (s.get("classes") or [])] if isinstance(s.get("classes"), list) else [],
|
|
"subclasses": [c.get("name") for c in (s.get("subclasses") or [])] if isinstance(s.get("subclasses"), list) else [],
|
|
}
|
|
|
|
|
|
def simplify_creature(c: dict) -> dict:
|
|
"""Extract key fields from a creature."""
|
|
return {
|
|
"key": c.get("key", ""),
|
|
"name": c.get("name", ""),
|
|
"size": c.get("size", {}).get("name") if isinstance(c.get("size"), dict) else c.get("size"),
|
|
"type": c.get("type", {}).get("name") if isinstance(c.get("type"), dict) else c.get("type"),
|
|
"alignment": c.get("alignment", ""),
|
|
"armor_class": c.get("armor_class", 0),
|
|
"hit_points": c.get("hit_points", 0),
|
|
"hit_dice": c.get("hit_dice", ""),
|
|
"speed": c.get("speed", {}),
|
|
"strength": c.get("strength", 10),
|
|
"dexterity": c.get("dexterity", 10),
|
|
"constitution": c.get("constitution", 10),
|
|
"intelligence": c.get("intelligence", 10),
|
|
"wisdom": c.get("wisdom", 10),
|
|
"charisma": c.get("charisma", 10),
|
|
"saving_throws": c.get("saving_throws", {}),
|
|
"skills": c.get("skills", {}),
|
|
"damage_vulnerabilities": c.get("damage_vulnerabilities", []),
|
|
"damage_resistances": c.get("damage_resistances", []),
|
|
"damage_immunities": c.get("damage_immunities", []),
|
|
"condition_immunities": c.get("condition_immunities", []),
|
|
"senses": c.get("senses", ""),
|
|
"languages": c.get("languages", ""),
|
|
"challenge_rating": c.get("challenge_rating", "0"),
|
|
"xp": c.get("xp", 0),
|
|
"traits": [{"name": t.get("name"), "desc": t.get("desc")} for t in (c.get("traits") or [])],
|
|
"actions": [{"name": a.get("name"), "desc": a.get("desc")} for a in (c.get("actions") or [])],
|
|
"legendary_actions": [{"name": a.get("name"), "desc": a.get("desc")} for a in (c.get("legendary_actions") or [])],
|
|
"description": c.get("desc", ""),
|
|
}
|
|
|
|
|
|
def simplify_class(c: dict) -> dict:
|
|
"""Extract key fields from a class."""
|
|
return {
|
|
"key": c.get("key", ""),
|
|
"name": c.get("name", ""),
|
|
"hit_dice": c.get("hit_dice", ""),
|
|
"description": c.get("desc", ""),
|
|
"spellcasting_ability": c.get("spellcasting_ability", {}).get("key") if isinstance(c.get("spellcasting_ability"), dict) else c.get("spellcasting_ability"),
|
|
"subclasses": [s.get("name") for s in (c.get("subclasses") or [])] if isinstance(c.get("subclasses"), list) else [],
|
|
}
|
|
|
|
|
|
def simplify_magic_item(m: dict) -> dict:
|
|
"""Extract key fields from a magic item."""
|
|
return {
|
|
"key": m.get("key", ""),
|
|
"name": m.get("name", ""),
|
|
"type": m.get("type", {}).get("name") if isinstance(m.get("type"), dict) else m.get("type"),
|
|
"rarity": m.get("rarity", {}).get("name") if isinstance(m.get("rarity"), dict) else m.get("rarity"),
|
|
"requires_attunement": m.get("requires_attunement", ""),
|
|
"description": m.get("desc", ""),
|
|
}
|
|
|
|
|
|
def simplify_item(i: dict) -> dict:
|
|
"""Extract key fields from equipment/item."""
|
|
cat = i.get("category") or {}
|
|
return {
|
|
"key": i.get("key", ""),
|
|
"name": i.get("name", ""),
|
|
"category": cat.get("name") if isinstance(cat, dict) else cat,
|
|
"cost": i.get("cost", ""),
|
|
"weight": i.get("weight", ""),
|
|
"description": i.get("desc", ""),
|
|
"armor_class": i.get("armor_class"),
|
|
"stealth_disadvantage": i.get("stealth_disadvantage", False),
|
|
"strength_requirement": i.get("strength_requirement"),
|
|
"damage": i.get("damage", {}).get("damage_dice") if isinstance(i.get("damage"), dict) else None,
|
|
"properties": [p.get("name") for p in (i.get("properties") or [])] if isinstance(i.get("properties"), list) else [],
|
|
}
|
|
|
|
|
|
SIMPLIFIERS = {
|
|
"spells": simplify_spell,
|
|
"creatures": simplify_creature,
|
|
"classes": simplify_class,
|
|
"magic-items": simplify_magic_item,
|
|
"equipment": simplify_item,
|
|
}
|
|
|
|
|
|
def main():
|
|
for name, first_url in SOURCES.items():
|
|
try:
|
|
raw_data = fetch_all(name, first_url)
|
|
simplifier = SIMPLIFIERS.get(name)
|
|
if simplifier:
|
|
simplified = [simplifier(item) for item in raw_data]
|
|
else:
|
|
simplified = raw_data
|
|
|
|
# Build index: name -> slug and key -> item
|
|
indexed = {
|
|
item.get("key"): item
|
|
for item in simplified
|
|
if item.get("key")
|
|
}
|
|
|
|
# Also build a name index
|
|
name_index = {}
|
|
for item in simplified:
|
|
name_lower = item.get("name", "").lower()
|
|
slug = item.get("key", "")
|
|
if name_lower:
|
|
name_index[name_lower] = slug
|
|
|
|
output = {
|
|
"count": len(simplified),
|
|
"results": simplified,
|
|
"_index": {
|
|
"by_key": {k: True for k in indexed.keys()},
|
|
"by_name": name_index,
|
|
}
|
|
}
|
|
|
|
# Remove _index for display but keep for reference
|
|
filepath = os.path.join(DATA_DIR, f"{name}.json")
|
|
with open(filepath, "w") as f:
|
|
json.dump(output, f, indent=2)
|
|
|
|
print(f"💾 Saved {len(simplified)} items to {filepath}")
|
|
|
|
# Also save raw data for debugging
|
|
raw_filepath = os.path.join(DATA_DIR, f"{name}_raw.json")
|
|
with open(raw_filepath, "w") as f:
|
|
json.dump({"count": len(raw_data), "results": raw_data}, f, indent=2)
|
|
print(f"💾 Raw backup saved to {raw_filepath}")
|
|
|
|
except Exception as e:
|
|
print(f"❌ Failed to fetch {name}: {e}")
|
|
sys.exit(1)
|
|
|
|
print("\n✨ All data fetched successfully!")
|
|
print(json.dumps({"status": "ok", "dir": DATA_DIR}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |