import unittest from fastapi.testclient import TestClient import main from db import connect, create SAMPLE_URL = "https://www.allrecipes.com/recipe/262696/cheese-omelette/" with open("tests/sample_files/recipes/cheese-omelette.html", "r", encoding="utf-8") as f: SAMPLE_HTML = f.read() class TestRecipesParseFromUrlIntegrationV2(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): self.conn = await connect(":memory:") await create(self.conn) async def override_get_db(): try: yield self.conn finally: pass main.app.dependency_overrides[main.get_db] = override_get_db self.client = TestClient(main.app) # Register user and create household r = self.client.post( "/api/v1/auth/register", json={"email": "scrape@test.com", "password": "pw", "displayName": "Scrape"}, ) assert r.status_code == 200, r.text token = r.json()["accessToken"] self.headers = {"Authorization": f"Bearer {token}"} r2 = self.client.post("/api/v1/households", headers=self.headers, json={"name": "H"}) assert r2.status_code == 200, r2.text self.slug = r2.json()["slug"] async def asyncTearDown(self): await self.conn.close() main.app.dependency_overrides.clear() def test_parse_from_url_with_mocked_html(self): # Monkeypatch httpx.AsyncClient.get used in scraper import recipes.scraping as scraping class DummyResp: def __init__(self, status_code=200, text=""): self.status_code = status_code self.text = text class DummyClient: def __init__(self, *args, **kwargs): pass async def __aenter__(self): return self async def __aexit__(self, exc_type, exc, tb): return False async def get(self, url, headers=None, follow_redirects=False): # Allow the scraper to call the base URL or an AMP fallback assert url == SAMPLE_URL or ( url.startswith(SAMPLE_URL) and ("output=amp" in url or url.endswith("/amp")) ) return DummyResp(200, SAMPLE_HTML) orig_client = scraping.httpx.AsyncClient scraping.httpx.AsyncClient = DummyClient try: r = self.client.post( f"/api/v1/households/{self.slug}/recipes/parse-from-url", headers=self.headers, json={"url": SAMPLE_URL}, ) assert r.status_code == 200, r.text data = r.json() # Now returns RecipeCreate shape assert "id" not in data assert "name" in data and "omelette" in data["name"].lower() assert isinstance(data.get("ingredients"), list) assert isinstance(data.get("imageUrls"), list) finally: scraping.httpx.AsyncClient = orig_client def test_parse_from_url_not_found(self): import recipes.scraping as scraping class DummyResp: def __init__(self, status_code=404, text=""): self.status_code = status_code self.text = text class DummyClient: def __init__(self, *args, **kwargs): pass async def __aenter__(self): return self async def __aexit__(self, exc_type, exc, tb): return False async def get(self, url, headers=None, follow_redirects=False): return DummyResp(404, "") orig_client = scraping.httpx.AsyncClient scraping.httpx.AsyncClient = DummyClient try: r = self.client.post( f"/api/v1/households/{self.slug}/recipes/parse-from-url", headers=self.headers, json={"url": SAMPLE_URL}, ) assert r.status_code == 422, r.text body = r.json() assert body.get("status") == 422 finally: scraping.httpx.AsyncClient = orig_client def test_parse_from_url_audab_460_then_amp_fallback(self): """Simulate an AUDAB 460 block on the canonical URL, then succeed on an AMP variant. Also verify we send browser-like headers including User-Agent and Accept-Encoding. """ import recipes.scraping as scraping calls = [] class DummyResp: def __init__(self, status_code=200, text=""): self.status_code = status_code self.text = text class DummyClient: def __init__(self, *args, **kwargs): pass async def __aenter__(self): return self async def __aexit__(self, exc_type, exc, tb): return False async def get(self, url, headers=None, follow_redirects=False): # record call calls.append((url, headers or {})) # First attempt to the canonical URL returns AUDAB 460 if url == SAMPLE_URL: # Ensure headers include browser-like values ua = (headers or {}).get("User-Agent", "") assert "Mozilla" in ua or "Chrome" in ua ae = (headers or {}).get("Accept-Encoding", "") assert "gzip" in ae # Simulate block return DummyResp(460, "AUDAB - Not Allowed") # Fallback attempt(s): '?output=amp' should succeed if url.startswith(SAMPLE_URL) and "output=amp" in url: return DummyResp(200, SAMPLE_HTML) # Any other path fails to ensure the code tries the intended fallback return DummyResp(404, "") orig_client = scraping.httpx.AsyncClient scraping.httpx.AsyncClient = DummyClient try: r = self.client.post( f"/api/v1/households/{self.slug}/recipes/parse-from-url", headers=self.headers, json={"url": SAMPLE_URL}, ) assert r.status_code == 200, r.text data = r.json() assert "name" in data and isinstance(data.get("ingredients"), list) # Assert we attempted the base URL then an AMP variant urls_called = [u for (u, _h) in calls] assert urls_called[0] == SAMPLE_URL assert any("output=amp" in u for u in urls_called[1:]) finally: scraping.httpx.AsyncClient = orig_client