134 lines
4.8 KiB
Python
134 lines
4.8 KiB
Python
|
"""
|
||
|
|
Manual harness to exercise recipes.parse_recipe against specific public URLs.
|
||
|
|
|
||
|
|
This WILL NOT run in CI or with `make test` — run it explicitly:
|
||
|
|
|
||
|
|
make install # first time
|
||
|
|
python -m scripts.manual_parse_recipes
|
||
|
|
|
||
|
|
Notes:
|
||
|
|
- Creates/bootstraps a local SQLite DB (./data/doof.sqlite) for lookups.
|
||
|
|
- Does not persist recipes; it just prints parsed results.
|
||
|
|
- Network access required to fetch pages.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
from types import SimpleNamespace
|
||
|
|
from typing import Any, Dict, List
|
||
|
|
|
||
|
|
from db import connect, create
|
||
|
|
from recipes import parse_recipe
|
||
|
|
from scripts.recipe_urls import URLS
|
||
|
|
import os
|
||
|
|
|
||
|
|
# This harness streams results as they complete and keeps per-URL logs grouped.
|
||
|
|
|
||
|
|
|
||
|
|
def _fmt(v) -> str:
|
||
|
|
if v is None:
|
||
|
|
return "-"
|
||
|
|
return str(v)
|
||
|
|
|
||
|
|
|
||
|
|
def _print_recipe(r) -> None:
|
||
|
|
print("\n=== Parsed Recipe ===")
|
||
|
|
print(f"name: {_fmt(getattr(r, 'name', None))}")
|
||
|
|
print(f"serves: {_fmt(getattr(r, 'serves', None))}")
|
||
|
|
print(f"link: {_fmt(getattr(r, 'link', None))}")
|
||
|
|
imgs = getattr(r, "image_urls", []) or []
|
||
|
|
if imgs:
|
||
|
|
print(f"images[0]: {imgs[0]}")
|
||
|
|
ings = getattr(r, "ingredients", []) or []
|
||
|
|
print(f"ingredients: {len(ings)}")
|
||
|
|
for ing in ings[:10]:
|
||
|
|
# Each is an Ingredient model with line/name/quantity/unit
|
||
|
|
n = getattr(ing, "name", "")
|
||
|
|
q = getattr(ing, "quantity", "")
|
||
|
|
u = getattr(ing, "unit", "")
|
||
|
|
line = getattr(ing, "line", "")
|
||
|
|
print(f" - {n} ({q} {u}) :: {line}")
|
||
|
|
|
||
|
|
|
||
|
|
async def _parse_one(conn, url: str, dump_dir: str | None) -> Dict[str, Any]:
|
||
|
|
# Minimal created_by object the parser expects (id, display_name)
|
||
|
|
created_by = SimpleNamespace(id=1, display_name="Manual Tester")
|
||
|
|
logs: List[str] = []
|
||
|
|
def _log(msg: str):
|
||
|
|
# Collect per-URL logs; we'll print them later in a grouped section
|
||
|
|
logs.append(msg)
|
||
|
|
try:
|
||
|
|
r = await parse_recipe(conn, created_by, url, log=_log, dump_dir=dump_dir)
|
||
|
|
except Exception as e:
|
||
|
|
return {"url": url, "recipe": None, "logs": logs, "error": str(e)}
|
||
|
|
return {"url": url, "recipe": r, "logs": logs, "error": None}
|
||
|
|
|
||
|
|
|
||
|
|
async def main() -> None:
|
||
|
|
conn = await connect()
|
||
|
|
# Ensure schema exists for product matching in ingredient parsing
|
||
|
|
await create(conn)
|
||
|
|
try:
|
||
|
|
dump_dir = os.environ.get("SCRAPER_DUMP_DIR") or None
|
||
|
|
concurrency = min(8, len(URLS))
|
||
|
|
sem = asyncio.Semaphore(concurrency)
|
||
|
|
# Consume all available slots and stagger release initially to avoid bursts
|
||
|
|
for _ in range(concurrency):
|
||
|
|
await sem.acquire()
|
||
|
|
# Release all slots with slight delays to avoid thundering herd
|
||
|
|
for i in range(concurrency):
|
||
|
|
asyncio.get_event_loop().call_later(i * 0.5, sem.release)
|
||
|
|
|
||
|
|
async def _worker(u: str):
|
||
|
|
async with sem:
|
||
|
|
return await _parse_one(conn, u, dump_dir)
|
||
|
|
|
||
|
|
start_time = asyncio.get_event_loop().time()
|
||
|
|
tasks = [asyncio.create_task(_worker(url)) for url in URLS]
|
||
|
|
|
||
|
|
|
||
|
|
# Stream results as they arrive; print per-URL blocks to avoid jumbled output
|
||
|
|
totals = {"total": 0, "parsed": 0, "errors": 0, "no_recipe": 0}
|
||
|
|
for task in asyncio.as_completed(tasks):
|
||
|
|
res = await task
|
||
|
|
url = res["url"]
|
||
|
|
totals["total"] += 1
|
||
|
|
print("\n==============================")
|
||
|
|
print(f"Result: {url}")
|
||
|
|
print("==============================")
|
||
|
|
if res.get("error"):
|
||
|
|
totals["errors"] += 1
|
||
|
|
print(f"ERROR: failed to parse {url}:")
|
||
|
|
print(f" {res['error']}")
|
||
|
|
r = res.get("recipe")
|
||
|
|
if r is None and not res.get("error"):
|
||
|
|
totals["no_recipe"] += 1
|
||
|
|
print(f"WARN: no recipe data found at {url}")
|
||
|
|
if r is not None:
|
||
|
|
totals["parsed"] += 1
|
||
|
|
_print_recipe(r)
|
||
|
|
|
||
|
|
# Logs
|
||
|
|
logs: List[str] = res.get("logs") or []
|
||
|
|
# Print logs for failures only to reduce noise; clip to a reasonable limit
|
||
|
|
if (r is None or res.get("error")) and logs:
|
||
|
|
clip = logs[:200]
|
||
|
|
print("-- logs --")
|
||
|
|
for line in clip:
|
||
|
|
print(f" dbg: {line}")
|
||
|
|
if len(logs) > len(clip):
|
||
|
|
print(f" .. ({len(logs) - len(clip)} more lines clipped) ..")
|
||
|
|
|
||
|
|
print("\n=== Summary ===")
|
||
|
|
print(f"Total URLs processed: {totals['total']}")
|
||
|
|
print(f"Successfully parsed: {totals['parsed']}")
|
||
|
|
print(f"Errors: {totals['errors']}")
|
||
|
|
print(f"No recipe found: {totals['no_recipe']}")
|
||
|
|
print(f"Time elapsed: {asyncio.get_event_loop().time() - start_time:.2f} seconds")
|
||
|
|
print(f"Remaining URLs: {len(URLS) - totals['total']}")
|
||
|
|
finally:
|
||
|
|
await conn.close()
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
asyncio.run(main())
|