munch-ease-backend/tests/test_recipes_parse_from_url_integration_v2.py

import unittest
from fastapi.testclient import TestClient

import main
from db import connect, create


SAMPLE_URL = "https://www.allrecipes.com/recipe/262696/cheese-omelette/"
with open("tests/sample_files/recipes/cheese-omelette.html", "r", encoding="utf-8") as f:
    SAMPLE_HTML = f.read()


class TestRecipesParseFromUrlIntegrationV2(unittest.IsolatedAsyncioTestCase):
    async def asyncSetUp(self):
        self.conn = await connect(":memory:")
        await create(self.conn)

        async def override_get_db():
            try:
                yield self.conn
            finally:
                pass

        main.app.dependency_overrides[main.get_db] = override_get_db
        self.client = TestClient(main.app)

        # Register user and create household
        r = self.client.post(
            "/api/v1/auth/register",
            json={"email": "scrape@test.com", "password": "pw", "displayName": "Scrape"},
        )
        assert r.status_code == 200, r.text
        token = r.json()["accessToken"]
        self.headers = {"Authorization": f"Bearer {token}"}
        r2 = self.client.post("/api/v1/households", headers=self.headers, json={"name": "H"})
        assert r2.status_code == 200, r2.text
        self.slug = r2.json()["slug"]

    async def asyncTearDown(self):
        await self.conn.close()
        main.app.dependency_overrides.clear()

    def test_parse_from_url_with_mocked_html(self):
        # Monkeypatch httpx.AsyncClient.get used in scraper
        import recipes.scraping as scraping

        class DummyResp:
            def __init__(self, status_code=200, text=""):
                self.status_code = status_code
                self.text = text

        class DummyClient:
            def __init__(self, *args, **kwargs):
                pass
            async def __aenter__(self):
                return self

            async def __aexit__(self, exc_type, exc, tb):
                return False

            async def get(self, url, headers=None, follow_redirects=False):
                # Allow the scraper to call the base URL or an AMP fallback
                assert url == SAMPLE_URL or (
                    url.startswith(SAMPLE_URL) and ("output=amp" in url or url.endswith("/amp"))
                )
                return DummyResp(200, SAMPLE_HTML)

        orig_client = scraping.httpx.AsyncClient
        scraping.httpx.AsyncClient = DummyClient
        try:
            r = self.client.post(
                f"/api/v1/households/{self.slug}/recipes/parse-from-url",
                headers=self.headers,
                json={"url": SAMPLE_URL},
            )
            assert r.status_code == 200, r.text
            data = r.json()
            # Now returns RecipeCreate shape
            assert "id" not in data
            assert "name" in data and "omelette" in data["name"].lower()
            assert isinstance(data.get("ingredients"), list)
            assert isinstance(data.get("imageUrls"), list)
        finally:
            scraping.httpx.AsyncClient = orig_client

    def test_parse_from_url_not_found(self):
        import recipes.scraping as scraping

        class DummyResp:
            def __init__(self, status_code=404, text=""):
                self.status_code = status_code
                self.text = text

        class DummyClient:
            def __init__(self, *args, **kwargs):
                pass
            async def __aenter__(self):
                return self

            async def __aexit__(self, exc_type, exc, tb):
                return False

            async def get(self, url, headers=None, follow_redirects=False):
                return DummyResp(404, "")

        orig_client = scraping.httpx.AsyncClient
        scraping.httpx.AsyncClient = DummyClient
        try:
            r = self.client.post(
                f"/api/v1/households/{self.slug}/recipes/parse-from-url",
                headers=self.headers,
                json={"url": SAMPLE_URL},
            )
            assert r.status_code == 422, r.text
            body = r.json()
            assert body.get("status") == 422
        finally:
            scraping.httpx.AsyncClient = orig_client

    def test_parse_from_url_audab_460_then_amp_fallback(self):
        """Simulate an AUDAB 460 block on the canonical URL, then succeed on an AMP variant.
        Also verify we send browser-like headers including User-Agent and Accept-Encoding.
        """
        import recipes.scraping as scraping

        calls = []

        class DummyResp:
            def __init__(self, status_code=200, text=""):
                self.status_code = status_code
                self.text = text

        class DummyClient:
            def __init__(self, *args, **kwargs):
                pass
            async def __aenter__(self):
                return self

            async def __aexit__(self, exc_type, exc, tb):
                return False

            async def get(self, url, headers=None, follow_redirects=False):
                # record call
                calls.append((url, headers or {}))
                # First attempt to the canonical URL returns AUDAB 460
                if url == SAMPLE_URL:
                    # Ensure headers include browser-like values
                    ua = (headers or {}).get("User-Agent", "")
                    assert "Mozilla" in ua or "Chrome" in ua
                    ae = (headers or {}).get("Accept-Encoding", "")
                    assert "gzip" in ae
                    # Simulate block
                    return DummyResp(460, "AUDAB - Not Allowed")
                # Fallback attempt(s): '?output=amp' should succeed
                if url.startswith(SAMPLE_URL) and "output=amp" in url:
                    return DummyResp(200, SAMPLE_HTML)
                # Any other path fails to ensure the code tries the intended fallback
                return DummyResp(404, "")

        orig_client = scraping.httpx.AsyncClient
        scraping.httpx.AsyncClient = DummyClient
        try:
            r = self.client.post(
                f"/api/v1/households/{self.slug}/recipes/parse-from-url",
                headers=self.headers,
                json={"url": SAMPLE_URL},
            )
            assert r.status_code == 200, r.text
            data = r.json()
            assert "name" in data and isinstance(data.get("ingredients"), list)
            # Assert we attempted the base URL then an AMP variant
            urls_called = [u for (u, _h) in calls]
            assert urls_called[0] == SAMPLE_URL
            assert any("output=amp" in u for u in urls_called[1:])
        finally:
            scraping.httpx.AsyncClient = orig_client