scraping improvements

recipes/__init__.py

@@ -23,16 +23,37 @@ from recipes.repository import (
     load_recipe_ingredients as load_recipe_ingredients,
     row_to_recipe as row_to_recipe,
 )
-from recipes.scraping import scrape_recipe_ldata as _scrape_recipe_ldata
+from recipes.scraping import (
+    scrape_recipe_ldata as _scrape_recipe_ldata,
+    scrape_recipe_ldata_from_html as _scrape_recipe_ldata_from_html,
+)
 
 
-async def parse_recipe(conn, created_by, url: str) -> Optional[Recipe]:
+async def parse_recipe(conn, created_by, url: str, log=None, dump_dir: str | None = None) -> Optional[Recipe]:
-    ldata = await _scrape_recipe_ldata(url)
+    """Parse a recipe from a URL. Returns None if parsing fails.
+
+    Accepts an optional log callable taking a single string argument; when provided,
+    the scraper will emit diagnostic messages useful for manual testing.
+    """
+    ldata = await _scrape_recipe_ldata(url, log=log, dump_dir=dump_dir)
+
     if ldata:
         return await _get_recipe_from_ldata(conn, url, ldata, created_by)
     return None
 
 
+async def parse_recipe_from_html(conn, created_by, base_url: str, html: str, log=None) -> Optional[Recipe]:
+    """Parse a recipe from raw HTML (offline). Returns None if parsing fails.
+
+    This mirrors parse_recipe() but uses already-downloaded HTML via the offline
+    scraper entrypoint. Useful for tests/assertions against saved snapshots.
+    """
+    ldata = _scrape_recipe_ldata_from_html(html, base_url, log=log)
+    if ldata:
+        return await _get_recipe_from_ldata(conn, base_url, ldata, created_by)
+    return None
+
+
 def find_yield(recipe_ldata: dict) -> int:
     if "recipeYield" in recipe_ldata:
         yield_vals = recipe_ldata["recipeYield"]

recipes/scraping.py

@@ -1,37 +1,275 @@
 import json
-from typing import Optional, Iterable, List
+from typing import Optional, Iterable, List, Tuple, Dict
 
 import httpx
 from bs4 import BeautifulSoup
+import html as _html
 
-# A realistic browser header profile improves success rates against some CDNs/bot protections.
+# POLICY: Strict ld+json-only recipe extraction
+# ---------------------------------------------
+# This scraper MUST NOT perform complex HTML-based ingredient parsing.
+# Only two things are allowed when parsing HTML:
+#   1) Discover additional, more-friendly variants of the same page (e.g., amp/print)
+#   2) Locate and parse <script type="application/ld+json"> blocks that contain a Recipe
+# Do NOT extract from application/json, __NEXT_DATA__, plugin markup, microdata, or generic
+# DOM heuristics. This keeps behavior predictable and aligned with sites' structured data.
+
+# Determine brotli support to avoid unreadable responses when the runtime lacks a decoder.
+_HAS_BROTLI = False
+try:  # brotli or brotlicffi
+    import brotli as _brotli  # type: ignore
+    _HAS_BROTLI = True
+except Exception:
+    try:
+        import brotlicffi as _brotlicffi  # type: ignore
+        _HAS_BROTLI = True
+    except Exception:
+        _HAS_BROTLI = False
+
+# Base headers shared across profiles; specific profiles add UA and optional fetch headers.
 DEFAULT_HEADERS = {
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
-    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Language": "en-US,en;q=0.8",
-    "Accept-Encoding": "gzip, deflate, br",
+    # Only advertise br when we can decode it, otherwise prefer gzip/deflate for reliability
+    "Accept-Encoding": "gzip, deflate, br" if _HAS_BROTLI else "gzip, deflate",
     "Connection": "keep-alive",
+}
+
+# Header profiles: start with an automation-forward identity, then fall back to browser-like
+HEADER_PROFILES = [
+    {
+        "name": "automation",
+        "headers": {
+            **DEFAULT_HEADERS,
+            "User-Agent": "MunchEaseRecipeBot/1.0 (+https://example.com/bot)",
+            "From": "bot@example.com",
+        },
+        "add_referer": False,
+    },
+    {
+        "name": "chrome-desktop",
+        "headers": {
+            **DEFAULT_HEADERS,
             "Upgrade-Insecure-Requests": "1",
             "Sec-Fetch-Dest": "document",
             "Sec-Fetch-Mode": "navigate",
             "Sec-Fetch-Site": "none",
             "Sec-Fetch-User": "?1",
-    # A modern desktop Chrome UA with platform tokens; not tied to any user data.
+            # Client hints
+            "sec-ch-ua": '"Chromium";v="127", "Not=A?Brand";v="24", "Google Chrome";v="127"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Linux"',
             "User-Agent": (
                 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
                 "Chrome/127.0.0.0 Safari/537.36"
             ),
-}
+        },
+        "add_referer": True,
+    },
+    {
+        "name": "chrome-mobile",
+        "headers": {
+            **DEFAULT_HEADERS,
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+            "sec-ch-ua": '"Chromium";v="127", "Not=A?Brand";v="24", "Google Chrome";v="127"',
+            "sec-ch-ua-mobile": "?1",
+            "sec-ch-ua-platform": '"Android"',
+            "User-Agent": (
+                "Mozilla/5.0 (Linux; Android 12; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/127.0.0.0 Mobile Safari/537.36"
+            ),
+        },
+        "add_referer": True,
+    },
+    {
+        "name": "chrome-full",
+        "headers": {
+            **DEFAULT_HEADERS,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+            "sec-ch-ua": '"Chromium";v="127", "Google Chrome";v="127", ";Not A Brand";v="99"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Linux"',
+            "sec-ch-ua-platform-version": '"6.8.0"',
+            "DNT": "1",
+            "Priority": "u=0, i",
+            "User-Agent": (
+                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/127.0.0.0 Safari/537.36"
+            ),
+        },
+        "add_referer": True,
+    },
+    {
+        "name": "safari-mac",
+        "headers": {
+            **DEFAULT_HEADERS,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": (
+                "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
+            ),
+        },
+        "add_referer": True,
+    },
+    {
+        "name": "firefox-desktop",
+        "headers": {
+            **DEFAULT_HEADERS,
+            "User-Agent": (
+                "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
+            ),
+        },
+        "add_referer": True,
+    },
+    {
+        "name": "safari-ios",
+        "headers": {
+            **DEFAULT_HEADERS,
+            "User-Agent": (
+                "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) "
+                "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
+            ),
+        },
+        "add_referer": True,
+    },
+    {
+        "name": "edge-desktop",
+        "headers": {
+            **DEFAULT_HEADERS,
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+            "sec-ch-ua": '"Chromium";v="127", "Not=A?Brand";v="24", "Microsoft Edge";v="127"',
+            "sec-ch-ua-mobile": "?0",
+            "sec-ch-ua-platform": '"Linux"',
+            "User-Agent": (
+                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0"
+            ),
+        },
+        "add_referer": True,
+    },
+]
+
+
+def looks_like_jsonld(html: str) -> bool:
+    """Lightweight check for presence of JSON-LD Recipe data in an HTML string.
+
+    We only check for ld+json tags or @context+schema.org signatures.
+    This is an optimization hint and does not parse JSON.
+    """
+    if not html:
+        return False
+    low = html.lower()
+    if "application/ld+json" in low:
+        return True
+    if "@context" in low and "schema.org" in low:
+        return True
+    return False
+
+
+def _force_safe_accept_encoding(headers: Dict[str, str]) -> Dict[str, str]:
+    """Return a copy of headers where Accept-Encoding excludes brotli.
+
+    Use this in contexts where brotli support may be missing to avoid unreadable payloads.
+    """
+    h = dict(headers)
+    h["Accept-Encoding"] = "gzip, deflate"
+    return h
+
+
+async def fetch_first_2xx_html(
+    client: httpx.AsyncClient,
+    url: str,
+    log=None,
+    safe_accept_encoding: bool = True,
+) -> Tuple[Optional[str], Dict[str, Optional[str]]]:
+    """Try header profiles against a URL and return the first 2xx HTML and metadata.
+
+    Returns (html, meta) where meta includes:
+      - status, final_url, content_type, content_encoding
+      - profile (name), request_headers (effective request headers)
+    If no profile returns 2xx, returns (None, meta_with_last_error_or_status).
+    """
+    def _log(msg: str) -> None:
+        if log:
+            try:
+                log(msg)
+            except Exception:
+                pass
+
+    from urllib.parse import urlparse
+    parsed = urlparse(url)
+    origin = f"{parsed.scheme}://{parsed.netloc}/"
+    last_meta: Dict[str, Optional[str]] = {
+        "status": None,
+        "final_url": None,
+        "content_type": None,
+        "content_encoding": None,
+        "profile": None,
+        "error": None,
+        "request_headers": None,
+    }
+    last_html: Optional[str] = None
+    for prof in HEADER_PROFILES:
+        headers = dict(prof["headers"])  # copy
+        if prof.get("add_referer"):
+            headers.setdefault("Referer", origin)
+        if safe_accept_encoding:
+            headers = _force_safe_accept_encoding(headers)
+        try:
+            resp = await client.get(url, headers=headers, follow_redirects=True)
+        except Exception as e:
+            last_meta.update({"error": f"{type(e).__name__}: {e}", "profile": prof["name"]})
+            _log(f"GET[{prof['name']}] {url} -> EXC {type(e).__name__}: {e}")
+            continue
+        _log(f"GET[{prof['name']}] {url} -> {resp.status_code}")
+        meta = {
+            "status": str(resp.status_code),
+            "final_url": str(resp.url),
+            "content_type": resp.headers.get("content-type"),
+            "content_encoding": resp.headers.get("content-encoding"),
+            "profile": prof["name"],
+            "error": None,
+            "request_headers": json.dumps(headers),
+        }
+        last_meta = meta
+        if 200 <= resp.status_code < 300:
+            html = resp.text
+            last_html = html
+            # Prefer early return if JSON-LD signature seems present
+            if looks_like_jsonld(html):
+                return html, meta
+            # else keep searching other profiles for a more suitable variant
+            continue
+    return last_html, last_meta
 
 
 def _is_recipe_ldata(ldata_node) -> bool:
-    if "@type" in ldata_node:
+    """Return True when a JSON-LD node represents a Recipe.
+
+    Handles cases where @type is a string or a list (order-insensitive).
+    """
+    if "@type" not in ldata_node:
+        return False
     typ = ldata_node["@type"]
+    if isinstance(typ, str):
+        return typ.lower() == "recipe"
     if isinstance(typ, list):
-            typ = typ[0]
+        for t in typ:
-
+            if isinstance(t, str) and t.lower() == "recipe":
-        if isinstance(typ, str) and typ.lower() == "recipe":
                 return True
-
     return False
 
 
@@ -43,6 +281,7 @@ def _fallback_urls(url: str) -> Iterable[str]:
     - add `?output=amp` if no existing query
     - add `&output=amp` if query exists
     - try `/amp` path suffix if not already present
+    - try `?amp=1` and bare `?amp` which some sites honor as AMP toggles
     """
     yield url
     try:
@@ -62,6 +301,42 @@ def _fallback_urls(url: str) -> Iterable[str]:
             amp2 = urlunparse(parsed._replace(path=amp_path))
             if amp2 != url:
                 yield amp2
+        # Additional common AMP toggles
+        q_amp = dict(parse_qsl(parsed.query, keep_blank_values=True))
+        if q_amp.get("amp") != "1":
+            q_amp["amp"] = "1"
+            amp1 = urlunparse(parsed._replace(query=urlencode(q_amp, doseq=True)))
+            if amp1 != url:
+                yield amp1
+        if "amp" not in (q_amp or {}):
+            bare = urlunparse(parsed._replace(query=(parsed.query + ("&" if parsed.query else "") + "amp")))
+            if bare != url:
+                yield bare
+
+        # HTTP scheme fallbacks when original is HTTPS (helps sites with misconfigured TLS)
+        if parsed.scheme == "https":
+            http_base = urlunparse(parsed._replace(scheme="http"))
+            if http_base != url:
+                yield http_base
+            # http + output=amp
+            q2 = dict(parse_qsl(parsed.query, keep_blank_values=True))
+            if q2.get("output") != "amp":
+                q2["output"] = "amp"
+            http_amp = urlunparse(parsed._replace(scheme="http", query=urlencode(q2, doseq=True)))
+            if http_amp != url:
+                yield http_amp
+            # http + amp=1
+            q3 = dict(parse_qsl(parsed.query, keep_blank_values=True))
+            if q3.get("amp") != "1":
+                q3["amp"] = "1"
+            http_amp1 = urlunparse(parsed._replace(scheme="http", query=urlencode(q3, doseq=True)))
+            if http_amp1 != url:
+                yield http_amp1
+            # http path amp
+            if not parsed.path.endswith("/amp"):
+                http_amp_path = urlunparse(parsed._replace(scheme="http", path=parsed.path.rstrip("/") + "/amp"))
+                if http_amp_path != url:
+                    yield http_amp_path
     except Exception:
         # Be conservative if URL parsing fails
         pass
@@ -70,222 +345,415 @@ def _fallback_urls(url: str) -> Iterable[str]:
 BLOCK_STATUSES = {403, 406, 429, 460}
 
 
-async def scrape_recipe_ldata(url: str) -> Optional[dict]:
+async def scrape_recipe_ldata(url: str, log=None, dump_dir: Optional[str] = None) -> Optional[dict]:
-    # Try the URL with browser-like headers and fallback strategies when blocked.
+    """Return best-effort recipe JSON-LD (or heuristic dict) for the URL.
-    async with httpx.AsyncClient() as client:
+
-        for candidate in _fallback_urls(url):
+    If 'log' is provided (callable taking a string), diagnostic messages are emitted
-            # Some CDNs prefer a referer; provide same-origin referer as a harmless hint.
+    during scraping. No environment toggles are used; behavior matches production.
-            headers = dict(DEFAULT_HEADERS)
+    """
-            headers.setdefault("Referer", candidate)
+
-            response = await client.get(candidate, headers=headers, follow_redirects=True)
+    def _log(msg: str) -> None:
-            if response.status_code in BLOCK_STATUSES:
+        if log:
-                # Try next fallback
+            try:
+                log(msg)
+            except Exception:
+                pass
+
+    try:
+        async with httpx.AsyncClient(http2=True, timeout=20.0) as client:
+            visited: set[str] = set()
+
+            # Queue of URLs to try, seeded with simple fallbacks; we'll append amp/print variants we discover.
+            to_try: List[str] = list(_fallback_urls(url))
+            while to_try:
+                candidate = to_try.pop(0)
+                if candidate in visited:
                     continue
-            if response.status_code >= 300:
+                visited.add(candidate)
-                # Try next fallback on non-2xx
+                # Try each header profile and attempt extraction per profile
+                any_2xx = False
+                from urllib.parse import urlparse, urlunparse
+                parsed = urlparse(candidate)
+                origin = f"{parsed.scheme}://{parsed.netloc}/"
+                for prof in HEADER_PROFILES:
+                    prof_name = prof["name"]
+                    headers = dict(prof["headers"])  # copy
+                    if prof.get("add_referer"):
+                        headers.setdefault("Referer", origin)
+                    try:
+                        resp = await client.get(candidate, headers=headers, follow_redirects=True)
+                    except Exception as e:
+                        _log(f"GET[{prof_name}] {candidate} -> EXC {type(e).__name__}: {e}")
+                        continue
+                    _log(f"GET[{prof_name}] {candidate} -> {resp.status_code}")
+                    if resp.status_code in BLOCK_STATUSES or resp.status_code >= 300:
+                        continue
+                    any_2xx = True
+                    html = resp.text
+                    soup = BeautifulSoup(html, "html.parser")
+                    # Optional debug dump
+                    if dump_dir:
+                        try:
+                            _dump_response(dump_dir, candidate, prof_name, resp)
+                        except Exception as e:
+                            _log(f"dump error: {type(e).__name__}: {e}")
+                    # Detect common bot protection pages early
+                    if (soup.title and "just a moment" in _extract_text(soup.title).lower()) or "challenge-platform" in html:
+                        _log("Detected Cloudflare-like challenge; trying next profile")
+                        continue
+                    # FRIENDLY DISCOVERY: follow declared amphtml and obvious print links
+                    try:
+                        # rel="amphtml"
+                        amp_link = None
+                        for link in soup.find_all("link"):
+                            rel = link.get("rel")
+                            href = link.get("href")
+                            if not href:
+                                continue
+                            if isinstance(rel, list) and any(r.lower() == "amphtml" for r in rel):
+                                amp_link = href
+                                break
+                            if isinstance(rel, str) and rel.lower() == "amphtml":
+                                amp_link = href
+                                break
+                        if amp_link:
+                            from urllib.parse import urljoin
+                            amp_abs = urljoin(candidate, amp_link)
+                            if amp_abs not in visited and amp_abs not in to_try:
+                                to_try.append(amp_abs)
+                                _log(f"discovered amphtml link: {amp_abs}")
+
+                        # print links (plugin or generic)
+                        from urllib.parse import urljoin, parse_qsl, urlencode
+                        for a in soup.find_all("a", href=True):
+                            href = a["href"]
+                            low = href.lower()
+                            if any(k in low for k in ["print", "wprm-print", "tasty-recipes-print", "/print/"]):
+                                absu = urljoin(candidate, href)
+                                if absu not in visited and absu not in to_try:
+                                    to_try.append(absu)
+                        # linked JSON-LD files
+                        for link in soup.find_all("link"):
+                            rel = link.get("rel")
+                            typ = (link.get("type") or link.get("as") or "").lower()
+                            href = link.get("href")
+                            if not href:
+                                continue
+                            if (isinstance(rel, list) and any(r.lower() == "alternate" for r in rel)) or (
+                                isinstance(rel, str) and rel.lower() == "alternate"
+                            ):
+                                if "ld+json" in typ:
+                                    from urllib.parse import urljoin
+                                    absu = urljoin(candidate, href)
+                                    if absu not in visited and absu not in to_try:
+                                        to_try.append(absu)
+                                        _log(f"discovered linked ld+json: {absu}")
+                        # query param prints: add print=1 once if missing
+                        parsed_url = parsed
+                        q_items = list(parse_qsl(parsed_url.query, keep_blank_values=True))
+                        has_print = any(k == "print" for k, _ in q_items)
+                        if not has_print:
+                            q_items.append(("print", "1"))
+                            print_url = urlunparse(parsed_url._replace(query=urlencode(q_items, doseq=True)))
+                            if print_url not in visited and print_url not in to_try:
+                                to_try.append(print_url)
+                                _log(f"queued print variant: {print_url}")
+                    except Exception as e:
+                        _log(f"discovery error: {type(e).__name__}: {e}")
+
+                    found = _extract_ldata_from_soup(soup, candidate, _log)
+                    if found:
+                        return found
+
+                if not any_2xx:
+                    _log("All header profiles blocked or non-2xx; trying next fallback")
                     continue
 
-            # Extract the recipe ld+json data from this candidate
+            _log("No recipe data found after all fallbacks")
-            soup = BeautifulSoup(response.text, "html.parser")
+            return None
-            for ld in soup.find_all("script", type="application/ld+json"):
+    except Exception as e:
+        _log(f"EXC during scraping: {type(e).__name__}: {e}")
+        return None
+
+
+def _extract_ldata_from_soup(soup: BeautifulSoup, candidate: str, log=None) -> Optional[dict]:
+    def _log(msg: str) -> None:
+        if log:
+            try:
+                log(msg)
+            except Exception:
+                pass
+
+    import re as _re
+
+    def _strip_to_json_candidate(text: str) -> str:
+        # Remove HTML/JS comment wrappers and CDATA markers
+        t = text.strip()
+        t = _re.sub(r"<!--|-->", "", t)
+        t = _re.sub(r"/\*.*?\*/", "", t, flags=_re.S)
+        t = _re.sub(r"(^|\s)//.*$", "", t, flags=_re.M)
+        # Trim to outermost JSON-like braces/brackets
+        first_brace = t.find("{")
+        first_brack = t.find("[")
+        starts = [i for i in [first_brace, first_brack] if i != -1]
+        if not starts:
+            return t
+        start = min(starts)
+        last_brace = t.rfind("}")
+        last_brack = t.rfind("]")
+        ends = [i for i in [last_brace, last_brack] if i != -1]
+        if not ends:
+            return t
+        end = max(ends)
+        return t[start : end + 1]
+
+    def _remove_trailing_commas(s: str) -> str:
+        prev = None
+        curr = s
+        # Iteratively remove trailing commas before } or ]
+        for _ in range(3):  # limit passes
+            prev = curr
+            curr = _re.sub(r",\s*([}\]])", r"\1", curr)
+            if curr == prev:
+                break
+        return curr
+
+    def _parse_json_lenient(text: str) -> Optional[object]:
+        # Try strict first
+        try:
+            return json.loads(text)
+        except Exception:
+            pass
+        # Clean wrappers and comments
+        t = _strip_to_json_candidate(text)
+        # Sometimes multiple roots are concatenated; try to form a list conservatively
+        if t.count("{") > 1 and "}\n{" in t:
+            parts = [p for p in t.split("\n") if p.strip()]
+            maybe = "[" + ",".join(parts) + "]"
+            try:
+                return json.loads(maybe)
+            except Exception:
+                pass
+        # Remove trailing commas
+        t2 = _remove_trailing_commas(t)
+        try:
+            return json.loads(t2)
+        except Exception:
+            pass
+        # As a last resort, if no double quotes exist but single quotes do, try naive conversion
+        if '"' not in t2 and "'" in t2:
+            t3 = t2.replace("'", '"')
+            try:
+                return json.loads(t3)
+            except Exception:
+                pass
+        return None
+    # 1) JSON-LD scripts (strict mode: only source of truth)
+    ld_nodes = soup.find_all("script", attrs={"type": _re.compile(r"ld\+json", _re.I)})
+    _log(f"Found {len(ld_nodes)} ld+json scripts")
+    for idx, ld in enumerate(ld_nodes):
         try:
             text = ld.text.strip()
-                    # Some sites embed multiple JSON objects without an array. Try to coerce if needed.
+            data = None
             try:
                 data = json.loads(text)
             except json.decoder.JSONDecodeError:
-                        # Attempt to split objects and wrap in a list (best-effort)
+                # Try lenient parsing strategies
-                        # Very conservative: only try if it looks like multiple root objects.
+                data = _parse_json_lenient(text)
-                        if text.count("{") > 1 and "}\n{" in text:
+                if data is not None:
-                            parts = [p for p in text.split("\n") if p.strip()]
+                    _log(f"ld[{idx}]: parsed with lenient JSON repair")
-                            maybe = "[" + ",".join(parts) + "]"
-                            data = json.loads(maybe)
                 else:
                     raise
-                    # _dump_json_data_to_log(data)
+
-                    if _is_recipe_ldata(data):
+            if isinstance(data, dict) and _is_recipe_ldata(data):
+                _log(f"ld[{idx}]: direct @type Recipe found")
                 return data
 
-                    if "@graph" in data:
+            if isinstance(data, dict) and "@graph" in data:
-                        for item in data["@graph"]:
+                for gidx, item in enumerate(data["@graph"]):
                     if _is_recipe_ldata(item):
+                        _log(f"ld[{idx}]: @graph item {gidx} is Recipe")
                         return item
 
             if isinstance(data, list):
-                        for item in data:
+                for lidx, item in enumerate(data):
                     if _is_recipe_ldata(item):
+                        _log(f"ld[{idx}]: list item {lidx} is Recipe")
                         return item
 
-                except (json.decoder.JSONDecodeError, KeyError):
+        except (json.decoder.JSONDecodeError, KeyError) as e:
-                    pass
+            _log(f"ld[{idx}]: JSON parse/Key error: {type(e).__name__}: {e}")
 
-            # Fallback 1: microdata (itemtype Recipe, itemprop recipeIngredient)
+    # 1b) JSON-LD within <noscript> blocks
-            md = _extract_recipe_from_microdata(soup)
+    try:
-            if md:
+        ns_count = 0
-                return md
+        for ns in soup.find_all("noscript"):
+            payload = ns.get_text(strip=False) or ns.decode_contents(formatter="html") or ""
+            if not payload:
+                continue
+            nsoup = BeautifulSoup(payload, "html.parser")
+            nodes = nsoup.find_all("script", attrs={"type": _re.compile(r"ld\+json", _re.I)})
+            ns_count += len(nodes)
+            for nidx, node in enumerate(nodes):
+                try:
+                    text = (node.text or node.get_text() or "").strip()
+                    if not text:
+                        continue
+                    data = _parse_json_lenient(text) or {}
+                    if isinstance(data, dict) and _is_recipe_ldata(data):
+                        _log(f"noscript.ld[{nidx}]: direct Recipe found")
+                        return data
+                    if isinstance(data, dict) and "@graph" in data:
+                        for gidx, item in enumerate(data.get("@graph") or []):
+                            if _is_recipe_ldata(item):
+                                _log(f"noscript.ld[{nidx}]: @graph item {gidx} is Recipe")
+                                return item
+                    if isinstance(data, list):
+                        for lidx, item in enumerate(data):
+                            if _is_recipe_ldata(item):
+                                _log(f"noscript.ld[{nidx}]: list item {lidx} is Recipe")
+                                return item
+                except Exception as e:
+                    _log(f"noscript.ld[{nidx}]: parse error: {type(e).__name__}: {e}")
+        if ns_count:
+            _log(f"noscript: scanned {ns_count} ld+json scripts")
+    except Exception as e:
+        _log(f"noscript scan error: {type(e).__name__}: {e}")
 
-            # Fallback 2: heuristic DOM extraction around "Ingredients" section
+    # 1c) JSON-LD by content signature in any script or HTML when type is missing
-            hd = _extract_recipe_heuristic(soup, candidate)
+    def _extract_json_objects_around(text: str) -> List[str]:
-            if hd:
+        out: List[str] = []
-                return hd
+        if not text:
+            return out
+        t = _html.unescape(text)
+        key = "@context"
+        pos = 0
+        while True:
+            at = t.find(key, pos)
+            if at == -1:
+                break
+            # find preceding '{'
+            lb = t.rfind("{", 0, at)
+            if lb == -1:
+                pos = at + len(key)
+                continue
+            i = lb
+            depth = 0
+            in_str = False
+            esc = False
+            end = -1
+            while i < len(t):
+                ch = t[i]
+                if in_str:
+                    if esc:
+                        esc = False
+                    elif ch == "\\":
+                        esc = True
+                    elif ch == '"':
+                        in_str = False
+                else:
+                    if ch == '"':
+                        in_str = True
+                    elif ch == '{':
+                        depth += 1
+                    elif ch == '}':
+                        depth -= 1
+                        if depth == 0:
+                            end = i
+                            break
+                i += 1
+            if end != -1:
+                out.append(t[lb:end+1])
+                pos = end + 1
+            else:
+                pos = at + len(key)
+        return out
 
+    def _try_parse_jsonld_candidates(cands: List[str], origin: str) -> Optional[dict]:
+        for cidx, cand in enumerate(cands):
+            data = _parse_json_lenient(cand)
+            if data is None:
+                continue
+            if isinstance(data, dict) and _is_recipe_ldata(data):
+                _log(f"{origin}[{cidx}]: Recipe found")
+                return data
+            if isinstance(data, dict) and "@graph" in data:
+                for gidx, item in enumerate(data.get("@graph") or []):
+                    if _is_recipe_ldata(item):
+                        _log(f"{origin}[{cidx}]: @graph item {gidx} is Recipe")
+                        return item
+            if isinstance(data, list):
+                for lidx, item in enumerate(data):
+                    if _is_recipe_ldata(item):
+                        _log(f"{origin}[{cidx}]: list item {lidx} is Recipe")
+                        return item
         return None
 
-    # Fallback return to satisfy static analysis
+    # Scan all <script> tags for JSON-LD signature when nothing found
+    try:
+        texts: List[str] = []
+        for sc in soup.find_all("script"):
+            txt = (sc.string or sc.get_text() or "").strip()
+            if not txt:
+                continue
+            if "@context" not in txt or "schema.org" not in txt:
+                continue
+            texts.append(txt)
+        if texts:
+            cands: List[str] = []
+            for t in texts:
+                cands.extend(_extract_json_objects_around(t))
+            found = _try_parse_jsonld_candidates(cands, origin="script-scan")
+            if found:
+                return found
+    except Exception as e:
+        _log(f"script content scan error: {type(e).__name__}: {e}")
+
+    # Finally, scan the full HTML string
+    try:
+        html_str = str(soup)
+        if "@context" in html_str and "schema.org" in html_str:
+            cands = _extract_json_objects_around(html_str)
+            found = _try_parse_jsonld_candidates(cands, origin="html-scan")
+            if found:
+                return found
+    except Exception as e:
+        _log(f"html scan error: {type(e).__name__}: {e}")
     return None
 
 
-def _dump_json_data_to_log(data: dict) -> str:
+def _dump_response(dump_dir: str, url: str, prof: str, resp: httpx.Response) -> None:
     import os
-    import re
+    from urllib.parse import urlparse
+    os.makedirs(dump_dir, exist_ok=True)
+    p = urlparse(url)
+    base = f"{p.netloc}{p.path}"
+    if not base or base.endswith("/"):
+        base += "index"
+    safe = base.replace("/", "_").replace("?", "_").replace("&", "_")
+    fname = f"{safe}__{prof}__{resp.status_code}.html"
+    meta = f"{safe}__{prof}__{resp.status_code}.meta"
+    fpath = os.path.join(dump_dir, fname)
+    mpath = os.path.join(dump_dir, meta)
+    with open(fpath, "w", encoding=resp.encoding or "utf-8", errors="ignore") as f:
+        f.write(resp.text)
+    with open(mpath, "w", encoding="utf-8") as f:
+        f.write(f"URL: {url}\n")
+        f.write(f"Profile: {prof}\n")
+        f.write(f"Status: {resp.status_code}\n")
+        f.write(f"Content-Type: {resp.headers.get('content-type','')}\n")
 
-    dir = "./data/dump"
-    if not os.path.exists(dir):
-        os.makedirs(dir)
 
-    prefix = "ldata_"
+def scrape_recipe_ldata_from_html(html: str, base_url: str, log=None) -> Optional[dict]:
-    suffix = ".json"
+    """Extract recipe JSON-LD from raw HTML (strict ld+json-only).
-    file_ids = [
+
-        int(re.findall(r"\d+", f)[0])
+    base_url is used for relative URL resolution and as a fallback name/link context.
-        for f in os.listdir(dir)
+    """
-        if re.match(prefix + r"\d+" + suffix, f)
+    soup = BeautifulSoup(html, "html.parser")
-    ]
+    return _extract_ldata_from_soup(soup, base_url, log)
-    id = max(file_ids) + 1 if file_ids else 0
+
-    filename = f"{prefix}{id}{suffix}"
-    full_path = os.path.join(dir, filename)
-    with open(full_path, "w") as f:
-        json.dump(data, f, indent=4)
-    return full_path
 
 
 def _extract_text(el) -> str:
     return " ".join(el.get_text(" ", strip=True).split()) if el else ""
 
-
-def _extract_recipe_from_microdata(soup: BeautifulSoup) -> Optional[dict]:
-    import re
-
-    candidates = []
-    for node in soup.find_all(attrs={"itemtype": re.compile(r"schema\.org/Recipe", re.I)}):
-        candidates.append(node)
-    # Some sites omit itemtype but use itemprop markers globally
-    if not candidates:
-        candidates = [soup]
-
-    def get_itemprop(node, prop) -> Optional[str]:
-        el = node.find(attrs={"itemprop": prop})
-        if not el:
-            return None
-        # Support <meta content>, <img src>, or text
-        if el.has_attr("content"):
-            return el["content"]
-        if el.has_attr("src"):
-            return el["src"]
-        if el.has_attr("href"):
-            return el["href"]
-        return _extract_text(el)
-
-    for node in candidates:
-        # Gather ingredients by itemprop
-        ingredients: List[str] = []
-        for prop in ("recipeIngredient", "ingredients"):
-            for ing_el in node.find_all(attrs={"itemprop": prop}):
-                t = _extract_text(ing_el)
-                if t:
-                    ingredients.append(t)
-
-        name = get_itemprop(node, "name") or get_itemprop(soup, "og:title") or None
-        image = (
-            get_itemprop(node, "image")
-            or get_itemprop(soup, "og:image")
-            or get_itemprop(soup, "twitter:image")
-        )
-        if image:
-            images = [image]
-        else:
-            images = []
-        ry = get_itemprop(node, "recipeYield") or get_itemprop(soup, "recipeYield")
-
-        # Minimal viability: at least one ingredient and a name
-        if ingredients and (name or len(ingredients) >= 2):
-            out = {"@type": "Recipe", "recipeIngredient": ingredients}
-            if name:
-                out["name"] = name
-            if images:
-                out["image"] = images
-            if ry:
-                out["recipeYield"] = ry
-            return out
-    return None
-
-
-def _extract_recipe_heuristic(soup: BeautifulSoup, page_url: str) -> Optional[dict]:
-    """Best-effort heuristic extraction when no structured data is present.
-
-    - Finds an Ingredients heading and the nearest list (ul/ol)
-    - Or any element with class/id containing 'ingredient' and list items
-    - Derives name from og:title or <title>
-    - Derives image from og:image if available
-    """
-    import re
-
-    # Try explicit ingredients container by class/id
-    candidates = []
-    for el in soup.find_all(True, attrs={"class": re.compile("ingredient", re.I)}):
-        candidates.append(el)
-    for el in soup.find_all(True, attrs={"id": re.compile("ingredient", re.I)}):
-        candidates.append(el)
-
-    def collect_list_items(node) -> List[str]:
-        items: List[str] = []
-        for ul in node.find_all(["ul", "ol"]):
-            for li in ul.find_all("li"):
-                t = _extract_text(li)
-                if t:
-                    items.append(t)
-        return items
-
-    ingredients: List[str] = []
-    for node in candidates:
-        ingredients = collect_list_items(node)
-        if ingredients:
-            break
-
-    # Try heading-based approach
-    if not ingredients:
-        for h in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
-            if "ingredient" in _extract_text(h).lower():
-                # Try next sibling lists or parent lists
-                sib = h.find_next_sibling()
-                tries = [sib, h.parent]
-                for t in tries:
-                    if not t:
-                        continue
-                    lst = collect_list_items(t)
-                    if lst:
-                        ingredients = lst
-                        break
-            if ingredients:
-                break
-
-    if not ingredients:
-        return None
-
-    # Name and image fallbacks
-    name = None
-    og_title = soup.find("meta", attrs={"property": "og:title"})
-    if og_title and og_title.has_attr("content"):
-        name = og_title["content"]
-    if not name and soup.title:
-        name = _extract_text(soup.title)
-
-    image = None
-    og_image = soup.find("meta", attrs={"property": "og:image"})
-    if og_image and og_image.has_attr("content"):
-        image = og_image["content"]
-
-    out = {"@type": "Recipe", "recipeIngredient": ingredients}
-    out["name"] = name or page_url
-    if image:
-        out["image"] = [image]
-    return out

scripts/generate_expected_from_snapshots.py

@@ -0,0 +1,64 @@
+"""
+Generate expected Recipe models from saved HTML snapshots (offline).
+
+Reads HTML files from tests/sample_files/recipes and writes expected
+Recipe JSONs into tests/sample_files/recipes/expected with matching slugs.
+
+Run:
+    python -m scripts.generate_expected_from_snapshots
+"""
+
+import json
+from pathlib import Path
+from typing import Optional
+from types import SimpleNamespace
+
+from db import connect, create
+from recipes import parse_recipe_from_html
+
+
+SNAP_DIR = Path("tests/sample_files/recipes")
+
+
+def _write_expected(slug: str, recipe) -> None:
+    # Write <slug>.recipe.json at SNAP_DIR root (canonical)
+    new_out = SNAP_DIR / f"{slug}.recipe.json"
+    if recipe is None:
+        new_out.write_text("null")
+    else:
+        new_out.write_text(recipe.model_dump_json(by_alias=True, indent=2))
+
+
+async def _run() -> None:
+    html_files = sorted(p for p in SNAP_DIR.glob("*.html"))
+    if not html_files:
+        print("no HTML snapshots found; run scripts.save_recipe_pages first")
+        return
+
+    # Minimal created_by object the parser expects (id, display_name)
+    created_by = SimpleNamespace(id=1, display_name="Snapshot Generator")
+    conn = await connect()
+    await create(conn)
+    try:
+        for p in html_files:
+            slug = p.stem
+            url = None
+            meta_path = SNAP_DIR / f"{slug}.meta.json"
+            if meta_path.exists():
+                try:
+                    meta = json.loads(meta_path.read_text())
+                    url = meta.get("final_url") or meta.get("url")
+                except Exception:
+                    pass
+            base_url = url or slug
+            html = p.read_text()
+            recipe = await parse_recipe_from_html(conn, created_by, base_url, html)
+            _write_expected(slug, recipe)
+            print(f"wrote expected for {slug}: {'ok' if recipe else 'none'}")
+    finally:
+        await conn.close()
+
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(_run())

scripts/manual_parse_recipes.py

@@ -0,0 +1,133 @@
+"""
+Manual harness to exercise recipes.parse_recipe against specific public URLs.
+
+This WILL NOT run in CI or with `make test` — run it explicitly:
+
+    make install  # first time
+    python -m scripts.manual_parse_recipes
+
+Notes:
+- Creates/bootstraps a local SQLite DB (./data/doof.sqlite) for lookups.
+- Does not persist recipes; it just prints parsed results.
+- Network access required to fetch pages.
+"""
+
+import asyncio
+from types import SimpleNamespace
+from typing import Any, Dict, List
+
+from db import connect, create
+from recipes import parse_recipe
+from scripts.recipe_urls import URLS
+import os
+
+# This harness streams results as they complete and keeps per-URL logs grouped.
+
+
+def _fmt(v) -> str:
+    if v is None:
+        return "-"
+    return str(v)
+
+
+def _print_recipe(r) -> None:
+    print("\n=== Parsed Recipe ===")
+    print(f"name:        {_fmt(getattr(r, 'name', None))}")
+    print(f"serves:      {_fmt(getattr(r, 'serves', None))}")
+    print(f"link:        {_fmt(getattr(r, 'link', None))}")
+    imgs = getattr(r, "image_urls", []) or []
+    if imgs:
+        print(f"images[0]:   {imgs[0]}")
+    ings = getattr(r, "ingredients", []) or []
+    print(f"ingredients: {len(ings)}")
+    for ing in ings[:10]:
+        # Each is an Ingredient model with line/name/quantity/unit
+        n = getattr(ing, "name", "")
+        q = getattr(ing, "quantity", "")
+        u = getattr(ing, "unit", "")
+        line = getattr(ing, "line", "")
+        print(f"  - {n} ({q} {u}) :: {line}")
+
+
+async def _parse_one(conn, url: str, dump_dir: str | None) -> Dict[str, Any]:
+    # Minimal created_by object the parser expects (id, display_name)
+    created_by = SimpleNamespace(id=1, display_name="Manual Tester")
+    logs: List[str] = []
+    def _log(msg: str):
+        # Collect per-URL logs; we'll print them later in a grouped section
+        logs.append(msg)
+    try:
+        r = await parse_recipe(conn, created_by, url, log=_log, dump_dir=dump_dir)
+    except Exception as e:
+        return {"url": url, "recipe": None, "logs": logs, "error": str(e)}
+    return {"url": url, "recipe": r, "logs": logs, "error": None}
+
+
+async def main() -> None:
+    conn = await connect()
+    # Ensure schema exists for product matching in ingredient parsing
+    await create(conn)
+    try:
+        dump_dir = os.environ.get("SCRAPER_DUMP_DIR") or None
+        concurrency = min(8, len(URLS))
+        sem = asyncio.Semaphore(concurrency)
+        # Consume all available slots and stagger release initially to avoid bursts
+        for _ in range(concurrency):
+            await sem.acquire()
+        # Release all slots with slight delays to avoid thundering herd
+        for i in range(concurrency):
+            asyncio.get_event_loop().call_later(i * 0.5, sem.release)
+
+        async def _worker(u: str):
+            async with sem:
+                return await _parse_one(conn, u, dump_dir)
+
+        start_time = asyncio.get_event_loop().time()
+        tasks = [asyncio.create_task(_worker(url)) for url in URLS]
+
+
+        # Stream results as they arrive; print per-URL blocks to avoid jumbled output
+        totals = {"total": 0, "parsed": 0, "errors": 0, "no_recipe": 0}
+        for task in asyncio.as_completed(tasks):
+            res = await task
+            url = res["url"]
+            totals["total"] += 1
+            print("\n==============================")
+            print(f"Result: {url}")
+            print("==============================")
+            if res.get("error"):
+                totals["errors"] += 1
+                print(f"ERROR: failed to parse {url}:")
+                print(f"  {res['error']}")
+            r = res.get("recipe")
+            if r is None and not res.get("error"):
+                totals["no_recipe"] += 1
+                print(f"WARN: no recipe data found at {url}")
+            if r is not None:
+                totals["parsed"] += 1
+                _print_recipe(r)
+
+            # Logs
+            logs: List[str] = res.get("logs") or []
+            # Print logs for failures only to reduce noise; clip to a reasonable limit
+            if (r is None or res.get("error")) and logs:
+                clip = logs[:200]
+                print("-- logs --")
+                for line in clip:
+                    print(f"  dbg: {line}")
+                if len(logs) > len(clip):
+                    print(f"  .. ({len(logs) - len(clip)} more lines clipped) ..")
+
+            print("\n=== Summary ===")
+            print(f"Total URLs processed: {totals['total']}")
+            print(f"Successfully parsed:  {totals['parsed']}")
+            print(f"Errors:               {totals['errors']}")
+            print(f"No recipe found:      {totals['no_recipe']}")
+            print(f"Time elapsed: {asyncio.get_event_loop().time() - start_time:.2f} seconds")
+            print(f"Remaining URLs:       {len(URLS) - totals['total']}")
+    finally:
+        await conn.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

scripts/recipe_urls.py

@@ -0,0 +1,37 @@
+"""
+Central list of recipe URLs used by manual harnesses and snapshot tools.
+
+Keep this list focused on stable, public pages across a variety of sites.
+"""
+
+URLS = [
+    # ld+json: YES (Recipe present under @graph)
+    "https://cookieandkate.com/favorite-fried-eggs-recipe/",
+    # ld+json: UNKNOWN (blocked/SSL issues in our environment)
+    "https://www.taste.com.au/recipes/classic-chewy-brownie/f0960a83-1fa9-4e3e-b616-a995182613e0",
+    # ld+json: PRESENT but MALFORMED (first ld+json is Recipe but not valid JSON)
+    "https://www.inspiredtaste.net/24412/cocoa-brownies-recipe/",
+    # ld+json: YES (Recipe present)
+    "https://www.simplyrecipes.com/recipes/french_toast/",
+    # Additional diverse sources
+    # ld+json: YES (Recipe present)
+    "https://pinchofyum.com/lemon-chicken-piccata-with-grilled-bread",
+    # ld+json: YES (Recipe present)
+    "https://www.bbcgoodfood.com/recipes/chicken-tikka-masala",
+    # ld+json: YES (Recipe present)
+    "https://www.recipetineats.com/beef-stroganoff/",
+    # ld+json: YES (Recipe present)
+    "https://sallysbakingaddiction.com/triple-chocolate-layer-cake/",
+    # ld+json: YES (Recipe present)
+    "https://downshiftology.com/recipes/shakshuka/",
+    # ld+json: UNKNOWN (403 blocked by CDN)
+    "https://damndelicious.net/2025/08/01/corn-salsa/",
+    # ld+json: UNKNOWN (403 blocked by CDN)
+    "https://www.thekitchn.com/chicken-tamale-pie-23752740",
+    # ld+json: YES (Recipe present)
+    "https://www.simplyrecipes.com/recipes/perfect_guacamole/",
+    # ld+json: YES (Recipe present)
+    "https://www.kingarthurbaking.com/recipes/banana-bread-recipe",
+    # ld+json: YES (Recipe present; multiple ld+json, only the first has Recipe)
+    "https://tasty.co/recipe/one-pot-garlic-parmesan-pasta",
+]

scripts/save_recipe_pages.py

@@ -0,0 +1,122 @@
+"""
+Snapshot recipe pages locally for offline inspection and future parsing tests.
+
+Run explicitly (not part of CI by default):
+
+    python -m scripts.save_recipe_pages
+
+This will download each URL in scripts/recipe_urls.py and save into:
+    tests/sample_files/recipes/
+
+For each URL, it writes:
+  - <slug>.html      The raw page HTML
+  - <slug>.meta.json JSON metadata (url, fetched_at, status, final_url, error)
+
+Notes:
+  - Uses the same browser-like headers as our scraper.
+  - Respects redirects; stores final_url.
+  - Skips writing HTML on hard errors but still writes a .meta.json with the error.
+  - Creates parent folders as needed.
+"""
+
+import asyncio
+import json
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Tuple
+
+import httpx
+
+from scripts.recipe_urls import URLS
+from recipes.scraping import DEFAULT_HEADERS, HEADER_PROFILES, _fallback_urls, fetch_first_2xx_html, looks_like_jsonld
+
+
+OUT_DIR = Path("tests/sample_files/recipes")
+
+
+def _safe_slug_from_url(url: str) -> str:
+    # Keep domain and last path segment(s) for readability; replace non-word with '-'
+    from urllib.parse import urlparse
+    p = urlparse(url)
+    host = p.netloc.replace(":", "-")
+    path = p.path.rstrip("/")
+    if not path:
+        seg = "index"
+    else:
+        # use last two segments if available to avoid collisions like /recipe/ vs /recipe-2/
+        parts = [s for s in path.split("/") if s]
+        seg = "-".join(parts[-2:]) if len(parts) >= 2 else parts[-1]
+    raw = f"{host}-{seg}".lower()
+    slug = re.sub(r"[^a-z0-9._-]+", "-", raw).strip("-")
+    return slug or "page"
+
+
+async def _fetch_best_html(client: httpx.AsyncClient, url: str) -> Tuple[str, Dict]:
+    meta: Dict = {
+        "url": url,
+        "fetched_at": datetime.now(timezone.utc).isoformat(),
+        "status": None,
+        "final_url": None,
+        "error": None,
+        "profile": None,
+        "content_type": None,
+        "content_encoding": None,
+        "candidate": None,
+    }
+    visited: set[str] = set()
+    candidates = list(_fallback_urls(url))
+    for cand in candidates:
+        if cand in visited:
+            continue
+        visited.add(cand)
+        html, prof_meta = await fetch_first_2xx_html(client, cand, safe_accept_encoding=True)
+        # Merge selected metadata
+        meta.update({
+            "status": int(prof_meta.get("status") or 0) or None,
+            "final_url": prof_meta.get("final_url"),
+            "content_type": prof_meta.get("content_type"),
+            "content_encoding": prof_meta.get("content_encoding"),
+            "profile": prof_meta.get("profile"),
+            "request_headers": prof_meta.get("request_headers"),
+            "candidate": cand,
+            "error": prof_meta.get("error"),
+        })
+        if html and looks_like_jsonld(html):
+            return html, meta
+        # If html is present, keep as a fallback in case later candidates fail
+        if html:
+            fallback_html = html
+            fallback_meta = dict(meta)
+            continue
+    # Nothing succeeded; return empty payload with last meta
+    return "", meta
+
+
+async def main() -> None:
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    async with httpx.AsyncClient(http2=True) as client:
+        sem = asyncio.Semaphore(6)
+
+        async def worker(u: str):
+            async with sem:
+                html, meta = await _fetch_best_html(client, u)
+                slug = _safe_slug_from_url(u)
+                html_path = OUT_DIR / f"{slug}.html"
+                meta_path = OUT_DIR / f"{slug}.meta.json"
+                try:
+                    # Always write metadata
+                    meta_path.write_text(json.dumps(meta, indent=2))
+                    # Only persist HTML on successful fetch
+                    if meta.get("status") and 200 <= int(meta["status"]) < 300 and html:
+                        html_path.write_text(html)
+                    print(f"saved: {u} -> {html_path.name} (status={meta.get('status')}, error={meta.get('error')})")
+                except Exception as e:
+                    print(f"failed to save for {u}: {type(e).__name__}: {e}")
+
+        await asyncio.gather(*(worker(u) for u in URLS))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/sample_files/recipes/README.md

@@ -0,0 +1,17 @@
+# Recipe page snapshots
+
+This folder contains raw HTML snapshots and metadata for public recipe pages, saved for offline inspection and to support unit tests without hitting third-party sites.
+
+Generated by:
+- `python -m scripts.save_recipe_pages` (downloads pages listed in `scripts/recipe_urls.py`)
+- `python -m scripts.generate_expected_from_snapshots` (parses HTML into structured recipe JSON)
+
+File layout per URL:
+- `<slug>.html` — raw page HTML (only for successful 2xx responses)
+- `<slug>.meta.json` — metadata with original URL, final URL, status code, fetch time, and any error
+- `<slug>.recipe.json` — expected parsed Recipe model (Pydantic-serialized, by_alias=True). This is the canonical expectation file used by offline tests.
+
+Notes:
+- Some domains may block requests (e.g., 403) or fail TLS verification. In those cases, only the `.meta.json` is written.
+- Keep `scripts/recipe_urls.py` curated to stable, public URLs to minimize churn.
+

tests/sample_files/recipes/cheese-omelette.recipe.json

@@ -0,0 +1,91 @@
+{
+  "id": -1,
+  "name": "Cheese Omelette",
+  "link": "cheese-omelette",
+  "serves": 1,
+  "imageUrls": [
+    "https://www.allrecipes.com/thmb/JS43mD2rA6_cCs9eTlXNGRHT5oQ=/1500x0/filters:no_upscale():max_bytes(150000):strip_icc()/5143634-ac5ad80b28f44c53bd0fd6d570f61f0d.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "eggs",
+      "line": "3 large eggs",
+      "unit": "Items",
+      "quantity": 3.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "milk",
+      "line": "1 tablespoon milk, or as needed",
+      "unit": "Tablespoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "salt and white pepper",
+      "line": "salt and freshly ground white pepper to taste",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "freshly ground",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "butter",
+      "line": "2 tablespoons butter",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "Emmentaler cheese",
+      "line": "0.25 cup shredded Emmentaler cheese",
+      "unit": "Cup",
+      "quantity": 0.25,
+      "preparation": "shredded",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.300361+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/cookieandkate.com-favorite-fried-eggs-recipe.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://cookieandkate.com/favorite-fried-eggs-recipe/",
+  "fetched_at": "2025-11-04T07:23:56.579020+00:00",
+  "status": 200,
+  "final_url": "https://cookieandkate.com/favorite-fried-eggs-recipe/",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "https://cookieandkate.com/favorite-fried-eggs-recipe/",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/cookieandkate.com-favorite-fried-eggs-recipe.recipe.json

@@ -0,0 +1,48 @@
+{
+  "id": -1,
+  "name": "Favorite Fried Eggs",
+  "link": "https://cookieandkate.com/favorite-fried-eggs-recipe/",
+  "serves": 1,
+  "imageUrls": [
+    "https://cookieandkate.com/images/2018/09/crispy-fried-egg-recipe-225x225.jpg",
+    "https://cookieandkate.com/images/2018/09/crispy-fried-egg-recipe-260x195.jpg",
+    "https://cookieandkate.com/images/2018/09/crispy-fried-egg-recipe-320x180.jpg",
+    "https://cookieandkate.com/images/2018/09/crispy-fried-egg-recipe.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "extra-virgin olive oil",
+      "line": "1 tablespoon extra-virgin olive oil",
+      "unit": "Tablespoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "egg",
+      "line": "1 egg",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.367540+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/damndelicious.net-01-corn-salsa.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://damndelicious.net/2025/08/01/corn-salsa/",
+  "fetched_at": "2025-11-04T07:23:56.680123+00:00",
+  "status": 403,
+  "final_url": "https://damndelicious.net/2025/08/01/corn-salsa/amp",
+  "error": null,
+  "profile": "edge-desktop",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "http://damndelicious.net/2025/08/01/corn-salsa/amp",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"Upgrade-Insecure-Requests\": \"1\", \"Sec-Fetch-Dest\": \"document\", \"Sec-Fetch-Mode\": \"navigate\", \"Sec-Fetch-Site\": \"none\", \"Sec-Fetch-User\": \"?1\", \"sec-ch-ua\": \"\\\"Chromium\\\";v=\\\"127\\\", \\\"Not=A?Brand\\\";v=\\\"24\\\", \\\"Microsoft Edge\\\";v=\\\"127\\\"\", \"sec-ch-ua-mobile\": \"?0\", \"sec-ch-ua-platform\": \"\\\"Linux\\\"\", \"User-Agent\": \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0\", \"Referer\": \"http://damndelicious.net/\"}"
+}

tests/sample_files/recipes/downshiftology.com-recipes-shakshuka.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://downshiftology.com/recipes/shakshuka/",
+  "fetched_at": "2025-11-04T07:23:56.673481+00:00",
+  "status": 200,
+  "final_url": "https://downshiftology.com/recipes/shakshuka/",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "https://downshiftology.com/recipes/shakshuka/",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/downshiftology.com-recipes-shakshuka.recipe.json

@@ -0,0 +1,178 @@
+{
+  "id": -1,
+  "name": "Shakshuka Recipe (Easy & Traditional)",
+  "link": "https://downshiftology.com/recipes/shakshuka/",
+  "serves": 6,
+  "imageUrls": [
+    "https://i2.wp.com/www.downshiftology.com/wp-content/uploads/2023/12/Shakshuka-main-1.jpg",
+    "https://downshiftology.com/wp-content/uploads/2023/12/Shakshuka-main-1-500x500.jpg",
+    "https://downshiftology.com/wp-content/uploads/2023/12/Shakshuka-main-1-500x375.jpg",
+    "https://downshiftology.com/wp-content/uploads/2023/12/Shakshuka-main-1-480x270.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "olive oil",
+      "line": "2 tablespoons olive oil",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "onion",
+      "line": "1  medium onion (diced)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "(diced)",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "red bell pepper",
+      "line": "1  red bell pepper (seeded and diced)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "(seeded and diced)",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "garlic",
+      "line": "4  garlic cloves (finely chopped)",
+      "unit": "Items",
+      "quantity": 4.0,
+      "preparation": "(finely chopped)",
+      "productId": 2,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 2,
+        "productId": "294517",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/294517/la-famiglia-garlic-bread",
+        "name": "La Famiglia Garlic Bread",
+        "quantity": 1,
+        "unit": "Loaf",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/294517.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/294517.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "paprika",
+      "line": "2 teaspoon paprika",
+      "unit": "Teaspoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "cumin",
+      "line": "1 teaspoon cumin",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "chili powder",
+      "line": "¼ teaspoon chili powder",
+      "unit": "Teaspoon",
+      "quantity": 0.25,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "whole peeled tomatoes",
+      "line": "1 (28-ounce can) whole peeled tomatoes",
+      "unit": "Ounce",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "eggs",
+      "line": "6  large eggs",
+      "unit": "Items",
+      "quantity": 6.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "salt and pepper",
+      "line": "salt and pepper (to taste)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "fresh cilantro",
+      "line": "1  small bunch fresh cilantro (chopped)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "(chopped)",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "fresh parsley",
+      "line": "1  small bunch fresh parsley (chopped)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "(chopped)",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.453255+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/pinchofyum.com-lemon-chicken-piccata-with-grilled-bread.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://pinchofyum.com/lemon-chicken-piccata-with-grilled-bread",
+  "fetched_at": "2025-11-04T07:23:56.582663+00:00",
+  "status": 200,
+  "final_url": "https://pinchofyum.com/lemon-chicken-piccata-with-grilled-bread",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "https://pinchofyum.com/lemon-chicken-piccata-with-grilled-bread",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/pinchofyum.com-lemon-chicken-piccata-with-grilled-bread.recipe.json

@@ -0,0 +1,144 @@
+{
+  "id": -1,
+  "name": "Lemon Chicken Piccata with Grilled Bread",
+  "link": "https://pinchofyum.com/lemon-chicken-piccata-with-grilled-bread",
+  "serves": 4,
+  "imageUrls": [
+    "https://pinchofyum.com/wp-content/uploads/Chicken-Piccata-Recipe-225x225.jpg",
+    "https://pinchofyum.com/wp-content/uploads/Chicken-Piccata-Recipe-260x195.jpg",
+    "https://pinchofyum.com/wp-content/uploads/Chicken-Piccata-Recipe-320x180.jpg",
+    "https://pinchofyum.com/wp-content/uploads/Chicken-Piccata-Recipe.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "boneless skinless chicken breasts",
+      "line": "1 pound boneless skinless chicken breasts",
+      "unit": "Pound",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "flour",
+      "line": "1/2 cup flour",
+      "unit": "Cup",
+      "quantity": 0.5,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "salt and pepper",
+      "line": "salt and pepper",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "LAND O LAKES® European Style Super Premium Salted Butter",
+      "line": "4 tablespoons LAND O LAKES® European Style Super Premium Salted Butter",
+      "unit": "Tablespoon",
+      "quantity": 4.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "olive oil",
+      "line": "2 tablespoons olive oil",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "white wine",
+      "line": "1/2 cup white wine",
+      "unit": "Cup",
+      "quantity": 0.5,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "chicken broth",
+      "line": "1 1/2 cups chicken broth",
+      "unit": "Cup",
+      "quantity": 1.5,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "lemon",
+      "line": "1 large lemon, sliced thinly (leave about 1/4 of the lemon intact for the juice)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "sliced thinly",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "jarred capers",
+      "line": "1/4 cup jarred capers",
+      "unit": "Cup",
+      "quantity": 0.25,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "fresh parsley",
+      "line": "fresh parsley",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.545344+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/sallysbakingaddiction.com-triple-chocolate-layer-cake.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://sallysbakingaddiction.com/triple-chocolate-layer-cake/",
+  "fetched_at": "2025-11-04T07:23:56.660551+00:00",
+  "status": 200,
+  "final_url": "https://sallysbakingaddiction.com/triple-chocolate-layer-cake/",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "https://sallysbakingaddiction.com/triple-chocolate-layer-cake/",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/sallysbakingaddiction.com-triple-chocolate-layer-cake.recipe.json

@@ -0,0 +1,282 @@
+{
+  "id": -1,
+  "name": "Deliciously Moist Chocolate Layer Cake",
+  "link": "https://sallysbakingaddiction.com/triple-chocolate-layer-cake/",
+  "serves": 12,
+  "imageUrls": [
+    "https://sallysbakingaddiction.com/wp-content/uploads/2013/04/triple-chocolate-cake-4-225x225.jpg",
+    "https://sallysbakingaddiction.com/wp-content/uploads/2013/04/triple-chocolate-cake-4-260x195.jpg",
+    "https://sallysbakingaddiction.com/wp-content/uploads/2013/04/triple-chocolate-cake-4-320x180.jpg",
+    "https://sallysbakingaddiction.com/wp-content/uploads/2013/04/triple-chocolate-cake-4.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "all-purpose flour",
+      "line": "1 and 3/4 cups (219g) all-purpose flour (spooned & leveled)",
+      "unit": "Cup",
+      "quantity": 1.75,
+      "preparation": "spooned &",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "unsweetened natural cocoa powder",
+      "line": "3/4 cup (62g) unsweetened natural cocoa powder",
+      "unit": "Cup",
+      "quantity": 0.75,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "granulated sugar",
+      "line": "1 and 3/4 cups (350g) granulated sugar",
+      "unit": "Cup",
+      "quantity": 1.75,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "baking soda",
+      "line": "2 teaspoons baking soda",
+      "unit": "Teaspoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "baking powder",
+      "line": "1 teaspoon baking powder",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "salt",
+      "line": "1 teaspoon salt",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": 5,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 5,
+        "productId": "33245",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/33245/saxa-iodised-table-salt-shaker",
+        "name": "Saxa Iodised Table Salt Shaker",
+        "quantity": 750,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/033245.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/033245.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "espresso powder",
+      "line": "2 teaspoons espresso powder (optional)",
+      "unit": "Teaspoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "vegetable oil",
+      "line": "1/2 cup (120ml) vegetable oil (or canola oil or melted coconut oil)",
+      "unit": "Cup",
+      "quantity": 0.5,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "eggs",
+      "line": "2 large eggs, at room temperature",
+      "unit": "Items",
+      "quantity": 2.0,
+      "preparation": "at room temperature",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "pure vanilla extract",
+      "line": "2 teaspoons pure vanilla extract",
+      "unit": "Teaspoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "buttermilk",
+      "line": "1 cup (240ml) buttermilk, at room temperature",
+      "unit": "Cup",
+      "quantity": 1.0,
+      "preparation": "at room temperature",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "strong hot coffee",
+      "line": "1 cup (240ml) freshly brewed strong hot coffee (regular or decaf)",
+      "unit": "Cup",
+      "quantity": 1.0,
+      "preparation": "freshly brewed",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "unsalted butter",
+      "line": "1 and 1/4 cups (282g) unsalted butter, softened to room temperature",
+      "unit": "Cup",
+      "quantity": 1.25,
+      "preparation": "softened to room temperature",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "confectioners’ sugar",
+      "line": "3 and 1/2 cups (420g) confectioners’ sugar",
+      "unit": "Cup",
+      "quantity": 3.5,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "unsweetened cocoa powder",
+      "line": "3/4 cup (62g) unsweetened cocoa powder (natural or dutch process)",
+      "unit": "Cup",
+      "quantity": 0.75,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "heavy cream",
+      "line": "3-5 Tablespoons (45-75ml) heavy cream (or half-and-half or milk), at room temperature",
+      "unit": "Tablespoon",
+      "quantity": 3.0,
+      "preparation": "at room temperature",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "salt",
+      "line": "1/4 teaspoon salt",
+      "unit": "Teaspoon",
+      "quantity": 0.25,
+      "preparation": "",
+      "productId": 5,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 5,
+        "productId": "33245",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/33245/saxa-iodised-table-salt-shaker",
+        "name": "Saxa Iodised Table Salt Shaker",
+        "quantity": 750,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/033245.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/033245.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "pure vanilla extract",
+      "line": "1 teaspoon pure vanilla extract",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "semi-sweet chocolate chips",
+      "line": "optional for decoration: semi-sweet chocolate chips",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.632664+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/tasty.co-recipe-one-pot-garlic-parmesan-pasta.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://tasty.co/recipe/one-pot-garlic-parmesan-pasta",
+  "fetched_at": "2025-11-04T07:23:56.855490+00:00",
+  "status": 200,
+  "final_url": "https://tasty.co/recipe/one-pot-garlic-parmesan-pasta",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=utf-8",
+  "content_encoding": "gzip",
+  "candidate": "https://tasty.co/recipe/one-pot-garlic-parmesan-pasta",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/tasty.co-recipe-one-pot-garlic-parmesan-pasta.recipe.json

@@ -0,0 +1,169 @@
+{
+  "id": -1,
+  "name": "One-Pot Garlic Parmesan Pasta Recipe by Tasty",
+  "link": "https://tasty.co/recipe/one-pot-garlic-parmesan-pasta",
+  "serves": 4,
+  "imageUrls": [
+    "https://img.buzzfeed.com/thumbnailer-prod-us-east-1/f69a7f4192b94d8395757b365ac6d866/GarlicParmPasta.jpg?resize=1200:*"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "unsalted butter",
+      "line": "2 tablespoons unsalted butter",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "garlic",
+      "line": "4 cloves garlic, minced",
+      "unit": "Items",
+      "quantity": 4.0,
+      "preparation": "minced",
+      "productId": 2,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 2,
+        "productId": "294517",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/294517/la-famiglia-garlic-bread",
+        "name": "La Famiglia Garlic Bread",
+        "quantity": 1,
+        "unit": "Loaf",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/294517.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/294517.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "chicken broth",
+      "line": "2 cups chicken broth",
+      "unit": "Cup",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "milk",
+      "line": "1 cup milk",
+      "unit": "Cup",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "fettuccine",
+      "line": "8 oz fettuccine",
+      "unit": "Ounce",
+      "quantity": 8.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "salt",
+      "line": "salt, to taste",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": 5,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 5,
+        "productId": "33245",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/33245/saxa-iodised-table-salt-shaker",
+        "name": "Saxa Iodised Table Salt Shaker",
+        "quantity": 750,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/033245.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/033245.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "pepper",
+      "line": "pepper, to taste",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": 6,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 6,
+        "productId": "75194",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/75194/mckenzie-s-pepper-black-ground",
+        "name": "Mckenzie's Pepper Black Ground",
+        "quantity": 100,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/075194.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/075194.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "parmesan cheese",
+      "line": "¼ cup grated parmesan cheese",
+      "unit": "Cup",
+      "quantity": 0.25,
+      "preparation": "grated",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "fresh parsley",
+      "line": "2 tablespoons fresh parsley, chopped",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.662183+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/www.bbcgoodfood.com-recipes-chicken-tikka-masala.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.bbcgoodfood.com/recipes/chicken-tikka-masala",
+  "fetched_at": "2025-11-04T07:23:56.582856+00:00",
+  "status": 200,
+  "final_url": "https://www.bbcgoodfood.com/recipes/chicken-tikka-masala",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=utf-8",
+  "content_encoding": "gzip",
+  "candidate": "https://www.bbcgoodfood.com/recipes/chicken-tikka-masala",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.bbcgoodfood.com-recipes-chicken-tikka-masala.recipe.json

@@ -0,0 +1,175 @@
+{
+  "id": -1,
+  "name": "Chicken tikka masala",
+  "link": "https://www.bbcgoodfood.com/recipes/chicken-tikka-masala",
+  "serves": 10,
+  "imageUrls": [
+    "https://images.immediate.co.uk/production/volatile/sites/30/2020/08/recipe-image-legacy-id-202451_12-50a0c95.jpg?resize=440,400"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "vegetable oil",
+      "line": "4 tbsp vegetable oil",
+      "unit": "Tablespoon",
+      "quantity": 4.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "butter",
+      "line": "25g butter",
+      "unit": "Gram",
+      "quantity": 25.0,
+      "preparation": "",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "onions",
+      "line": "4 onions roughly chopped",
+      "unit": "Items",
+      "quantity": 4.0,
+      "preparation": "roughly chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "chicken tikka masala paste",
+      "line": "6 tbsp chicken tikka masala paste (use shop-bought or make your own – see recipe, below)",
+      "unit": "Tablespoon",
+      "quantity": 6.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "red peppers",
+      "line": "2 red peppers deseeded and cut into chunks",
+      "unit": "Items",
+      "quantity": 2.0,
+      "preparation": "deseeded and cut into chunks",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "boneless, skinless chicken breasts",
+      "line": "8 boneless, skinless chicken breasts cut into 2.5cm cubes",
+      "unit": "Items",
+      "quantity": 8.0,
+      "preparation": "cut into 2.5 cm cubes",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "chopped tomatoes",
+      "line": "2 x 400g cans  chopped tomatoes",
+      "unit": "Gram",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "tomato purée",
+      "line": "4 tbsp tomato purée",
+      "unit": "Tablespoon",
+      "quantity": 4.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "mango chutney",
+      "line": "2-3 tbsp  mango chutney",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "double cream",
+      "line": "150ml double cream",
+      "unit": "Items",
+      "quantity": 150.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "natural yogurt",
+      "line": "150ml natural  yogurt",
+      "unit": "Items",
+      "quantity": 150.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "coriander leaves",
+      "line": "chopped  coriander leaves, to serve",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.711728+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/www.inspiredtaste.net-24412-cocoa-brownies-recipe.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.inspiredtaste.net/24412/cocoa-brownies-recipe/",
+  "fetched_at": "2025-11-04T07:23:56.582248+00:00",
+  "status": 200,
+  "final_url": "https://www.inspiredtaste.net/24412/cocoa-brownies-recipe/",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "https://www.inspiredtaste.net/24412/cocoa-brownies-recipe/",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.inspiredtaste.net-24412-cocoa-brownies-recipe.recipe.json

@@ -0,0 +1 @@
+null

tests/sample_files/recipes/www.kingarthurbaking.com-recipes-banana-bread-recipe.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.kingarthurbaking.com/recipes/banana-bread-recipe",
+  "fetched_at": "2025-11-04T07:23:56.767749+00:00",
+  "status": 200,
+  "final_url": "https://www.kingarthurbaking.com/recipes/banana-bread-recipe",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "https://www.kingarthurbaking.com/recipes/banana-bread-recipe",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.kingarthurbaking.com-recipes-banana-bread-recipe.recipe.json

@@ -0,0 +1,199 @@
+{
+  "id": -1,
+  "name": "Banana Bread",
+  "link": "https://www.kingarthurbaking.com/recipes/banana-bread-recipe",
+  "serves": 18,
+  "imageUrls": [
+    "https://www.kingarthurbaking.com/sites/default/files/recipe_legacy/5-3-large.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "unsalted butter",
+      "line": "8 tablespoons (113g) unsalted butter, at cool room temperature",
+      "unit": "Tablespoon",
+      "quantity": 8.0,
+      "preparation": "at cool room temperature",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "light brown sugar or dark brown sugar",
+      "line": "2/3 cup (142g) light brown sugar or dark brown sugar, packed",
+      "unit": "Cup",
+      "quantity": 0.667,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "King Arthur Pure Vanilla Extract",
+      "line": "1 teaspoon King Arthur Pure Vanilla Extract",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "cinnamon",
+      "line": "1 teaspoon cinnamon",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "nutmeg",
+      "line": "1/4 teaspoon nutmeg",
+      "unit": "Teaspoon",
+      "quantity": 0.25,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "baking soda",
+      "line": "1 teaspoon baking soda",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "baking powder",
+      "line": "1 teaspoon baking powder",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "table salt",
+      "line": "1 teaspoon table salt",
+      "unit": "Teaspoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "bananas",
+      "line": "1 1/2 cups (340g) bananas, mashed",
+      "unit": "Cup",
+      "quantity": 1.5,
+      "preparation": "mashed",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "apricot jam or orange marmalade",
+      "line": "3 tablespoons (64g) apricot jam or orange marmalade, optional but tasty",
+      "unit": "Tablespoon",
+      "quantity": 3.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "honey",
+      "line": "1/4 cup (85g) honey",
+      "unit": "Cup",
+      "quantity": 0.25,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "eggs",
+      "line": "2 large eggs",
+      "unit": "Items",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "King Arthur Unbleached All-Purpose Flour",
+      "line": "2 1/4 cups (270g) King Arthur Unbleached All-Purpose Flour",
+      "unit": "Cup",
+      "quantity": 2.25,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "walnuts",
+      "line": "1/2 cup (57g) chopped walnuts, optional",
+      "unit": "Cup",
+      "quantity": 0.5,
+      "preparation": "chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.834444+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/www.recipetineats.com-beef-stroganoff.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.recipetineats.com/beef-stroganoff/",
+  "fetched_at": "2025-11-04T07:23:56.628726+00:00",
+  "status": 200,
+  "final_url": "https://www.recipetineats.com/beef-stroganoff/",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=UTF-8",
+  "content_encoding": "gzip",
+  "candidate": "https://www.recipetineats.com/beef-stroganoff/",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.recipetineats.com-beef-stroganoff.recipe.json

@@ -0,0 +1,178 @@
+{
+  "id": -1,
+  "name": "Beef Stroganoff",
+  "link": "https://www.recipetineats.com/beef-stroganoff/",
+  "serves": 4,
+  "imageUrls": [
+    "https://www.recipetineats.com/tachyon/2018/01/Beef-Stroganoff_2-1-1.jpg",
+    "https://www.recipetineats.com/tachyon/2018/01/Beef-Stroganoff_2-1-1.jpg?resize=500%2C500",
+    "https://www.recipetineats.com/tachyon/2018/01/Beef-Stroganoff_2-1-1.jpg?resize=500%2C375",
+    "https://www.recipetineats.com/tachyon/2018/01/Beef-Stroganoff_2-1-1.jpg?resize=480%2C270"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "scotch fillet steak / boneless rib eye",
+      "line": "600 g / 1.2 lb   scotch fillet steak / boneless rib eye ((Note 1))",
+      "unit": "Gram",
+      "quantity": 600.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "vegetable oil",
+      "line": "2 tbsp vegetable oil (, divided)",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "(, divided)",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "onion",
+      "line": "1  large onion ((or 2 small onions), sliced)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "sliced",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "mushrooms",
+      "line": "300 g / 10 oz   mushrooms (, sliced (not too thin))",
+      "unit": "Gram",
+      "quantity": 300.0,
+      "preparation": ", sliced",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "butter",
+      "line": "40 g / 3 tbsp   butter",
+      "unit": "Gram",
+      "quantity": 40.0,
+      "preparation": "",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "flour",
+      "line": "2 tbsp flour ((Note 2))",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "beef broth",
+      "line": "2 cups / 500 ml   beef broth (, preferably salt reduced)",
+      "unit": "Cup",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Dijon mustard",
+      "line": "1 tbsp Dijon mustard",
+      "unit": "Tablespoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "sour cream",
+      "line": "150 ml / 2/3 cup   sour cream",
+      "unit": "Cup",
+      "quantity": 150.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Salt and pepper",
+      "line": "Salt and pepper",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "pasta or egg noodles of choice",
+      "line": "250 - 300 g / 8 - 10 oz   pasta or egg noodles of choice ((Note 3))",
+      "unit": "Gram",
+      "quantity": 250.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "chives",
+      "line": "Chopped chives (, for garnish (optional))",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "Chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.900537+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/www.simplyrecipes.com-recipes-french_toast.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.simplyrecipes.com/recipes/french_toast/",
+  "fetched_at": "2025-11-04T07:23:56.582449+00:00",
+  "status": 200,
+  "final_url": "https://www.simplyrecipes.com/recipes/french_toast/",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html;charset=utf-8",
+  "content_encoding": "gzip",
+  "candidate": "https://www.simplyrecipes.com/recipes/french_toast/",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.simplyrecipes.com-recipes-french_toast.recipe.json

@@ -0,0 +1,139 @@
+{
+  "id": -1,
+  "name": "The Best French Toast",
+  "link": "https://www.simplyrecipes.com/recipes/french_toast/",
+  "serves": 4,
+  "imageUrls": [
+    "https://www.simplyrecipes.com/thmb/34kTh59L8NjsXOB8nqw3hdYIKbs=/1500x0/filters:no_upscale():max_bytes(150000):strip_icc()/Simply-Recipes-Best-French-Toast-LEAD-4-ce3d4ce3d69b4c79a7bb3d9b83c8c3fc.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "eggs",
+      "line": "4 eggs",
+      "unit": "Items",
+      "quantity": 4.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "milk",
+      "line": "2/3 cup milk",
+      "unit": "Cup",
+      "quantity": 0.667,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "cinnamon",
+      "line": "2 teaspoons cinnamon",
+      "unit": "Teaspoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "2-day-old bread",
+      "line": "8 thick slices 2-day-old bread (better if slightly stale)",
+      "unit": "Items",
+      "quantity": 8.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Butter",
+      "line": "Butter (can sub vegetable oil)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "Maple syrup",
+      "line": "Maple syrup",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "orange zest",
+      "line": "2 teaspoons freshly grated orange zest",
+      "unit": "Teaspoon",
+      "quantity": 2.0,
+      "preparation": "freshly grated",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Triple Sec",
+      "line": "1/4 cup Triple Sec",
+      "unit": "Cup",
+      "quantity": 0.25,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Fresh berries",
+      "line": "Fresh berries",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:29.962968+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/www.simplyrecipes.com-recipes-perfect_guacamole.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.simplyrecipes.com/recipes/perfect_guacamole/",
+  "fetched_at": "2025-11-04T07:23:56.744212+00:00",
+  "status": 200,
+  "final_url": "https://www.simplyrecipes.com/recipes/perfect_guacamole/",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html;charset=utf-8",
+  "content_encoding": "gzip",
+  "candidate": "https://www.simplyrecipes.com/recipes/perfect_guacamole/",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.simplyrecipes.com-recipes-perfect_guacamole.recipe.json

@@ -0,0 +1,161 @@
+{
+  "id": -1,
+  "name": "How to Make the Best Guacamole",
+  "link": "https://www.simplyrecipes.com/recipes/perfect_guacamole/",
+  "serves": 4,
+  "imageUrls": [
+    "https://www.simplyrecipes.com/thmb/J4kA2m6jKMgkQwZhG-RYpjZBeFQ=/1500x0/filters:no_upscale():max_bytes(150000):strip_icc()/Guacamole-LEAD-6-2-64cfcca253c8421dad4e3fad830219f6.jpg"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "ripe avocados",
+      "line": "2 ripe avocados",
+      "unit": "Items",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "salt",
+      "line": "1/4 teaspoon salt, plus more to taste",
+      "unit": "Teaspoon",
+      "quantity": 0.25,
+      "preparation": "",
+      "productId": 5,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 5,
+        "productId": "33245",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/33245/saxa-iodised-table-salt-shaker",
+        "name": "Saxa Iodised Table Salt Shaker",
+        "quantity": 750,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/033245.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/033245.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "fresh lime or lemon juice",
+      "line": "1 tablespoon fresh lime or lemon juice",
+      "unit": "Tablespoon",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "red onion or thinly sliced green onion",
+      "line": "2-4 tablespoons minced red onion or thinly sliced green onion",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "minced",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "serrano (or jalapeño) chiles",
+      "line": "1-2 serrano (or jalapeño) chiles, stems and seeds removed, minced",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "stems and seeds removed, minced",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "cilantro",
+      "line": "2 tablespoons cilantro (leaves and tender stems), finely chopped",
+      "unit": "Tablespoon",
+      "quantity": 2.0,
+      "preparation": "finely chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "black pepper",
+      "line": "Pinch freshly ground black pepper",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "freshly ground",
+      "productId": 6,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 6,
+        "productId": "75194",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/75194/mckenzie-s-pepper-black-ground",
+        "name": "Mckenzie's Pepper Black Ground",
+        "quantity": 100,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/075194.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/075194.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "ripe tomato",
+      "line": "1/2 ripe tomato, chopped (optional)",
+      "unit": "Items",
+      "quantity": 0.5,
+      "preparation": "chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Red radish or jicama slices",
+      "line": "Red radish or jicama slices for garnish (optional)",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Tortilla chips",
+      "line": "Tortilla chips , to serve",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:30.070234+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/sample_files/recipes/www.taste.com.au-classic-chewy-brownie-f0960a83-1fa9-4e3e-b616-a995182613e0.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.taste.com.au/recipes/classic-chewy-brownie/f0960a83-1fa9-4e3e-b616-a995182613e0",
+  "fetched_at": "2025-11-04T07:23:56.581964+00:00",
+  "status": 403,
+  "final_url": "http://www.taste.com.au/recipes/classic-chewy-brownie/f0960a83-1fa9-4e3e-b616-a995182613e0/amp",
+  "error": "SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate (_ssl.c:1016)",
+  "profile": "edge-desktop",
+  "content_type": "text/html",
+  "content_encoding": null,
+  "candidate": "http://www.taste.com.au/recipes/classic-chewy-brownie/f0960a83-1fa9-4e3e-b616-a995182613e0/amp",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.thekitchn.com-chicken-tamale-pie-23752740.meta.json

@@ -0,0 +1,12 @@
+{
+  "url": "https://www.thekitchn.com/chicken-tamale-pie-23752740",
+  "fetched_at": "2025-11-04T07:23:56.686401+00:00",
+  "status": 200,
+  "final_url": "https://www.thekitchn.com/chicken-tamale-pie-23752740",
+  "error": null,
+  "profile": "automation",
+  "content_type": "text/html; charset=utf-8",
+  "content_encoding": "gzip",
+  "candidate": "https://www.thekitchn.com/chicken-tamale-pie-23752740",
+  "request_headers": "{\"Accept\": \"text/html,application/xhtml+xml;q=0.9,*/*;q=0.8\", \"Accept-Language\": \"en-US,en;q=0.8\", \"Accept-Encoding\": \"gzip, deflate\", \"Connection\": \"keep-alive\", \"User-Agent\": \"MunchEaseRecipeBot/1.0 (+https://example.com/bot)\", \"From\": \"bot@example.com\"}"
+}

tests/sample_files/recipes/www.thekitchn.com-chicken-tamale-pie-23752740.recipe.json

@@ -0,0 +1,211 @@
+{
+  "id": -1,
+  "name": "Cheesy Chicken Tamale Pie",
+  "link": "https://www.thekitchn.com/chicken-tamale-pie-23752740",
+  "serves": 6,
+  "imageUrls": [
+    "https://cdn.apartmenttherapy.info/image/upload/f_jpg,q_auto:eco,c_fill,g_auto,w_1500,ar_16:9/tk%2Fphoto%2F2025%2F10-2025%2F2025-10-chicken-tamale-pie%2Fchicken-tamale-pie-0",
+    "https://cdn.apartmenttherapy.info/image/upload/f_jpg,q_auto:eco,c_fill,g_auto,w_1500,ar_4:3/tk%2Fphoto%2F2025%2F10-2025%2F2025-10-chicken-tamale-pie%2Fchicken-tamale-pie-0",
+    "https://cdn.apartmenttherapy.info/image/upload/f_jpg,q_auto:eco,c_fill,g_auto,w_1500,ar_1:1/tk%2Fphoto%2F2025%2F10-2025%2F2025-10-chicken-tamale-pie%2Fchicken-tamale-pie-0"
+  ],
+  "ingredients": [
+    {
+      "id": -1,
+      "name": "unsalted butter",
+      "line": "3 tablespoons unsalted butter, divided",
+      "unit": "Tablespoon",
+      "quantity": 3.0,
+      "preparation": "divided",
+      "productId": 4,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 4,
+        "productId": "712251",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
+        "name": "Western Star Unsalted Butter Chef's Choice",
+        "quantity": 500,
+        "unit": "g",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "corn muffin mix",
+      "line": "1 (8.5-ounce) box corn muffin mix, such as Jiffy",
+      "unit": "Ounce",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "egg",
+      "line": "1  large egg",
+      "unit": "Items",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "sour cream",
+      "line": "1/2 cup sour cream, plus more for serving",
+      "unit": "Cup",
+      "quantity": 0.5,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "scallions",
+      "line": "4  medium scallions, thinly sliced (about 1/2 cup), plus more for garnish",
+      "unit": "Cup",
+      "quantity": 4.0,
+      "preparation": "thinly sliced",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "garlic",
+      "line": "2 cloves garlic, minced",
+      "unit": "Items",
+      "quantity": 2.0,
+      "preparation": "minced",
+      "productId": 2,
+      "recipeId": null,
+      "mealId": null,
+      "product": {
+        "id": 2,
+        "productId": "294517",
+        "shopCode": "woolworths",
+        "link": "https://www.woolworths.com.au/shop/productdetails/294517/la-famiglia-garlic-bread",
+        "name": "La Famiglia Garlic Bread",
+        "quantity": 1,
+        "unit": "Loaf",
+        "imgSmall": "https://cdn0.woolworths.media/content/wowproductimages/small/294517.jpg",
+        "imgLarge": "https://cdn0.woolworths.media/content/wowproductimages/large/294517.jpg"
+      }
+    },
+    {
+      "id": -1,
+      "name": "chili powder",
+      "line": "2 teaspoons chili powder",
+      "unit": "Teaspoon",
+      "quantity": 2.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "ground cumin",
+      "line": "1/2 teaspoon ground cumin",
+      "unit": "Teaspoon",
+      "quantity": 0.5,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "chicken",
+      "line": "3 cups shredded, cooked chicken (from 1/2 rotisserie chicken, about 10 ounces)",
+      "unit": "Cup",
+      "quantity": 3.0,
+      "preparation": "shredded, cooked",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "red enchilada sauce",
+      "line": "1 (10-ounce) can red enchilada sauce, or 1 1/4 cups homemade enchilada sauce",
+      "unit": "Ounce",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "frozen corn kernels",
+      "line": "1 cup frozen corn kernels, preferably fire-roasted (do not thaw)",
+      "unit": "Cup",
+      "quantity": 1.0,
+      "preparation": "",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "pickled jalapeños",
+      "line": "1/2 cup drained sliced pickled jalapeños, coarsely chopped, plus more for serving",
+      "unit": "Cup",
+      "quantity": 0.5,
+      "preparation": "drained sliced, coarsely chopped",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "sharp cheddar cheese",
+      "line": "4 ounces sharp cheddar cheese, shredded (about 1 cup)",
+      "unit": "Ounce",
+      "quantity": 4.0,
+      "preparation": "shredded",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    },
+    {
+      "id": -1,
+      "name": "Monterey Jack cheese",
+      "line": "4 ounces shredded Monterey Jack cheese, shredded (about 1 cup)",
+      "unit": "Ounce",
+      "quantity": 4.0,
+      "preparation": "shredded",
+      "productId": -1,
+      "recipeId": null,
+      "mealId": null,
+      "product": null
+    }
+  ],
+  "basedOnRecipe": null,
+  "dateCreated": "2025-11-04T18:24:30.117818+11:00",
+  "createdById": 1,
+  "createdBy": {
+    "id": 1,
+    "displayName": "Snapshot Generator"
+  },
+  "dateHidden": null,
+  "hiddenById": null,
+  "hiddenBy": null
+}

tests/test_recipes_offline_expected.py

@@ -0,0 +1,92 @@
+import asyncio
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from db import connect, create
+from recipes import parse_recipe_from_html
+
+
+SNAP_DIR = Path("tests/sample_files/recipes")
+
+
+def _iter_expected_files():
+    return sorted(SNAP_DIR.glob("*.recipe.json"))
+
+
+def _load_expected(path: Path):
+    # New-style file contains the model JSON directly or null
+    return json.loads(path.read_text())
+
+
+def _expected_subset(expected: dict):
+    # expected is serialized with by_alias=True, so image_urls is imageUrls
+    name = expected.get("name")
+    link = expected.get("link")
+    serves = expected.get("serves")
+    image_urls = expected.get("imageUrls") or []
+    ingredients = expected.get("ingredients") or []
+    ing_lines = [ing.get("line") for ing in ingredients]
+    return {
+        "name": name,
+        "link": link,
+        "serves": serves,
+        "image_urls": image_urls,
+        "ingredient_lines": ing_lines,
+    }
+
+
+def _actual_subset(model):
+    # Stable, assertion-friendly projection
+    return {
+        "name": model.name,
+        "link": model.link,
+        "serves": model.serves,
+        "image_urls": list(model.image_urls or []),
+        "ingredient_lines": [ing.line for ing in (model.ingredients or [])],
+    }
+
+
+@pytest.mark.parametrize("exp_path", _iter_expected_files())
+def test_offline_parse_matches_expected(exp_path: Path):
+    async def _run():
+        slug = exp_path.stem
+        expected = _load_expected(exp_path)
+
+        html_path = SNAP_DIR / f"{slug}.html"
+        assert html_path.exists(), f"snapshot HTML missing for {slug}"
+        html = html_path.read_text()
+
+        # Discover base URL from meta if available
+        meta_path = SNAP_DIR / f"{slug}.meta.json"
+        base_url = slug
+        if meta_path.exists():
+            try:
+                meta = json.loads(meta_path.read_text())
+                base_url = meta.get("final_url") or meta.get("url") or slug
+            except Exception:
+                pass
+
+        created_by = SimpleNamespace(id=1, display_name="Snapshot Generator")
+        conn = await connect()
+        await create(conn)
+        try:
+            model = await parse_recipe_from_html(conn, created_by, base_url, html)
+        finally:
+            await conn.close()
+
+        if expected is None:
+            assert model is None, f"expected no parse for {slug}, but got a model"
+            return
+
+        assert model is not None, f"expected a model for {slug}, got None"
+
+        exp_sub = _expected_subset(expected)
+        act_sub = _actual_subset(model)
+
+        # Compare key fields; ingredients compare by line only for stability
+        assert act_sub == exp_sub
+
+    asyncio.run(_run())

tests/test_recipes_parse_from_url_integration_v2.py

@@ -50,6 +50,8 @@ class TestRecipesParseFromUrlIntegrationV2(unittest.IsolatedAsyncioTestCase):
                 self.text = text
 
         class DummyClient:
+            def __init__(self, *args, **kwargs):
+                pass
             async def __aenter__(self):
                 return self
 
@@ -90,6 +92,8 @@ class TestRecipesParseFromUrlIntegrationV2(unittest.IsolatedAsyncioTestCase):
                 self.text = text
 
         class DummyClient:
+            def __init__(self, *args, **kwargs):
+                pass
             async def __aenter__(self):
                 return self
 
@@ -127,6 +131,8 @@ class TestRecipesParseFromUrlIntegrationV2(unittest.IsolatedAsyncioTestCase):
                 self.text = text
 
         class DummyClient:
+            def __init__(self, *args, **kwargs):
+                pass
             async def __aenter__(self):
                 return self
 

tests/test_recipes_parse_from_url_v2.py

@@ -40,11 +40,11 @@ class TestRecipesParseFromUrlV2(unittest.IsolatedAsyncioTestCase):
         # Monkeypatch scraper to avoid network
         import recipes.scraping as scraping
 
-        async def fake_scrape(url: str):
+        async def fake_scrape(url: str, log=None, dump_dir=None):
             assert url == "https://example.com/recipe"
             return {"@type": "Recipe", "name": "Example", "recipeIngredient": ["2 eggs"]}
 
-        async def fake_scrape_none(url: str):
+        async def fake_scrape_none(url: str, log=None, dump_dir=None):
             return None
 
         # Success case