feat: D&D SRD 5.2 API — FastAPI app with flat JSON caching
- Data fetched from Open5e API: 3,215 items (339 spells, 330 creatures, 24 classes, 2,319 magic items, 203 equipment) - FastAPI app with API key auth (X-API-Key header or ?api_key= param) - Sliding window rate limiting (60 req/min, 10K req/day) - Dice rolling endpoint (e.g., /api/dice/roll?spec=2d20+5) - Full-text search across all resource types - Pagination, filtering (name, level, school, class, etc.) - Admin CLI for API key management - nginx + systemd service ready for deployment
This commit is contained in:
parent
f62803bc7e
commit
f6858e6ea1
14 changed files with 58679 additions and 0 deletions
225
scripts/fetch_data.py
Normal file
225
scripts/fetch_data.py
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch all SRD 5.2 data from Open5e API and cache as flat JSON files.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data")
|
||||
SRD_FILTER = "document__key__in=srd-2024"
|
||||
|
||||
SOURCES = {
|
||||
"spells": f"https://api.open5e.com/v2/spells/?limit=100&{SRD_FILTER}",
|
||||
"creatures": f"https://api.open5e.com/v2/creatures/?limit=100&{SRD_FILTER}",
|
||||
"classes": f"https://api.open5e.com/v2/classes/?limit=100&{SRD_FILTER}",
|
||||
"magic-items": "https://api.open5e.com/v2/magicitems/?limit=100",
|
||||
"equipment": f"https://api.open5e.com/v2/items/?limit=100&{SRD_FILTER}",
|
||||
}
|
||||
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
|
||||
|
||||
def fetch_page(url: str, retries: int = 3) -> dict:
|
||||
"""Fetch a single page from the API with retries."""
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Cupcake-DnD-API/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
return data
|
||||
except Exception as e:
|
||||
if attempt < retries - 1:
|
||||
wait = 2 ** attempt
|
||||
print(f" ⚠ Retry {attempt + 1}/{retries} after {wait}s: {e}")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def fetch_all(name: str, first_url: str) -> list:
|
||||
"""Fetch all pages of a resource, returning combined results."""
|
||||
print(f"\n📦 Fetching {name}...")
|
||||
all_results = []
|
||||
url = first_url
|
||||
page = 0
|
||||
|
||||
while url:
|
||||
page += 1
|
||||
print(f" 📄 Page {page}: {url[:80]}...")
|
||||
data = fetch_page(url)
|
||||
results = data.get("results", [])
|
||||
all_results.extend(results)
|
||||
print(f" Got {len(results)} items (total: {len(all_results)})")
|
||||
url = data.get("next")
|
||||
|
||||
print(f"✅ {name}: {len(all_results)} total items")
|
||||
return all_results
|
||||
|
||||
|
||||
def simplify_spell(s: dict) -> dict:
|
||||
"""Extract key fields from a spell."""
|
||||
return {
|
||||
"key": s.get("key", "").replace("srd-2024_", ""),
|
||||
"name": s.get("name", ""),
|
||||
"level": s.get("level", 0),
|
||||
"school": s.get("school", {}).get("name") if isinstance(s.get("school"), dict) else s.get("school"),
|
||||
"casting_time": s.get("casting_time", ""),
|
||||
"range": s.get("range", ""),
|
||||
"components": {"material": s.get("material"), "verbal": s.get("verbal"), "somatic": s.get("somatic")},
|
||||
"duration": s.get("duration", ""),
|
||||
"concentration": s.get("concentration", False),
|
||||
"ritual": s.get("ritual", False),
|
||||
"description": s.get("desc", ""),
|
||||
"higher_level": s.get("higher_level", ""),
|
||||
"classes": [c.get("name") for c in (s.get("classes") or [])] if isinstance(s.get("classes"), list) else [],
|
||||
"subclasses": [c.get("name") for c in (s.get("subclasses") or [])] if isinstance(s.get("subclasses"), list) else [],
|
||||
}
|
||||
|
||||
|
||||
def simplify_creature(c: dict) -> dict:
|
||||
"""Extract key fields from a creature."""
|
||||
return {
|
||||
"key": c.get("key", ""),
|
||||
"name": c.get("name", ""),
|
||||
"size": c.get("size", {}).get("name") if isinstance(c.get("size"), dict) else c.get("size"),
|
||||
"type": c.get("type", {}).get("name") if isinstance(c.get("type"), dict) else c.get("type"),
|
||||
"alignment": c.get("alignment", ""),
|
||||
"armor_class": c.get("armor_class", 0),
|
||||
"hit_points": c.get("hit_points", 0),
|
||||
"hit_dice": c.get("hit_dice", ""),
|
||||
"speed": c.get("speed", {}),
|
||||
"strength": c.get("strength", 10),
|
||||
"dexterity": c.get("dexterity", 10),
|
||||
"constitution": c.get("constitution", 10),
|
||||
"intelligence": c.get("intelligence", 10),
|
||||
"wisdom": c.get("wisdom", 10),
|
||||
"charisma": c.get("charisma", 10),
|
||||
"saving_throws": c.get("saving_throws", {}),
|
||||
"skills": c.get("skills", {}),
|
||||
"damage_vulnerabilities": c.get("damage_vulnerabilities", []),
|
||||
"damage_resistances": c.get("damage_resistances", []),
|
||||
"damage_immunities": c.get("damage_immunities", []),
|
||||
"condition_immunities": c.get("condition_immunities", []),
|
||||
"senses": c.get("senses", ""),
|
||||
"languages": c.get("languages", ""),
|
||||
"challenge_rating": c.get("challenge_rating", "0"),
|
||||
"xp": c.get("xp", 0),
|
||||
"traits": [{"name": t.get("name"), "desc": t.get("desc")} for t in (c.get("traits") or [])],
|
||||
"actions": [{"name": a.get("name"), "desc": a.get("desc")} for a in (c.get("actions") or [])],
|
||||
"legendary_actions": [{"name": a.get("name"), "desc": a.get("desc")} for a in (c.get("legendary_actions") or [])],
|
||||
"description": c.get("desc", ""),
|
||||
}
|
||||
|
||||
|
||||
def simplify_class(c: dict) -> dict:
|
||||
"""Extract key fields from a class."""
|
||||
return {
|
||||
"key": c.get("key", ""),
|
||||
"name": c.get("name", ""),
|
||||
"hit_dice": c.get("hit_dice", ""),
|
||||
"description": c.get("desc", ""),
|
||||
"spellcasting_ability": c.get("spellcasting_ability", {}).get("key") if isinstance(c.get("spellcasting_ability"), dict) else c.get("spellcasting_ability"),
|
||||
"subclasses": [s.get("name") for s in (c.get("subclasses") or [])] if isinstance(c.get("subclasses"), list) else [],
|
||||
}
|
||||
|
||||
|
||||
def simplify_magic_item(m: dict) -> dict:
|
||||
"""Extract key fields from a magic item."""
|
||||
return {
|
||||
"key": m.get("key", ""),
|
||||
"name": m.get("name", ""),
|
||||
"type": m.get("type", {}).get("name") if isinstance(m.get("type"), dict) else m.get("type"),
|
||||
"rarity": m.get("rarity", {}).get("name") if isinstance(m.get("rarity"), dict) else m.get("rarity"),
|
||||
"requires_attunement": m.get("requires_attunement", ""),
|
||||
"description": m.get("desc", ""),
|
||||
}
|
||||
|
||||
|
||||
def simplify_item(i: dict) -> dict:
|
||||
"""Extract key fields from equipment/item."""
|
||||
return {
|
||||
"key": i.get("key", ""),
|
||||
"name": i.get("name", ""),
|
||||
"type": i.get("type", {}).get("name") if isinstance(i.get("type"), dict) else i.get("type"),
|
||||
"cost": i.get("cost", ""),
|
||||
"weight": i.get("weight", ""),
|
||||
"description": i.get("desc", ""),
|
||||
"armor_class": i.get("armor_class"),
|
||||
"stealth_disadvantage": i.get("stealth_disadvantage", False),
|
||||
"strength_requirement": i.get("strength_requirement"),
|
||||
"damage": i.get("damage", {}).get("damage_dice") if isinstance(i.get("damage"), dict) else None,
|
||||
"properties": [p.get("name") for p in (i.get("properties") or [])] if isinstance(i.get("properties"), list) else [],
|
||||
}
|
||||
|
||||
|
||||
SIMPLIFIERS = {
|
||||
"spells": simplify_spell,
|
||||
"creatures": simplify_creature,
|
||||
"classes": simplify_class,
|
||||
"magic-items": simplify_magic_item,
|
||||
"equipment": simplify_item,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
for name, first_url in SOURCES.items():
|
||||
try:
|
||||
raw_data = fetch_all(name, first_url)
|
||||
simplifier = SIMPLIFIERS.get(name)
|
||||
if simplifier:
|
||||
simplified = [simplifier(item) for item in raw_data]
|
||||
else:
|
||||
simplified = raw_data
|
||||
|
||||
# Build index: name -> slug and key -> item
|
||||
indexed = {
|
||||
item.get("key"): item
|
||||
for item in simplified
|
||||
if item.get("key")
|
||||
}
|
||||
|
||||
# Also build a name index
|
||||
name_index = {}
|
||||
for item in simplified:
|
||||
name_lower = item.get("name", "").lower()
|
||||
slug = item.get("key", "")
|
||||
if name_lower:
|
||||
name_index[name_lower] = slug
|
||||
|
||||
output = {
|
||||
"count": len(simplified),
|
||||
"results": simplified,
|
||||
"_index": {
|
||||
"by_key": {k: True for k in indexed.keys()},
|
||||
"by_name": name_index,
|
||||
}
|
||||
}
|
||||
|
||||
# Remove _index for display but keep for reference
|
||||
filepath = os.path.join(DATA_DIR, f"{name}.json")
|
||||
with open(filepath, "w") as f:
|
||||
json.dump(output, f, indent=2)
|
||||
|
||||
print(f"💾 Saved {len(simplified)} items to {filepath}")
|
||||
|
||||
# Also save raw data for debugging
|
||||
raw_filepath = os.path.join(DATA_DIR, f"{name}_raw.json")
|
||||
with open(raw_filepath, "w") as f:
|
||||
json.dump({"count": len(raw_data), "results": raw_data}, f, indent=2)
|
||||
print(f"💾 Raw backup saved to {raw_filepath}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to fetch {name}: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
print("\n✨ All data fetched successfully!")
|
||||
print(json.dumps({"status": "ok", "dir": DATA_DIR}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue