improved scraper

recipes/scraping.py

@@ -2,6 +2,22 @@ from bs4 import BeautifulSoup
 import httpx
 import json
 
+HEADERS = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'DNT': '1',
+    'Sec-GPC': '1',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Priority': 'u=1',
+    'Pragma': 'no-cache',
+    'Cache-Control': 'no-cache',
+}
+
 def _is_recipe_ldata(ldata_node):
     if '@type' in ldata_node:
         typ = ldata_node['@type']
@@ -14,16 +30,11 @@ def _is_recipe_ldata(ldata_node):
     return None
 
 async def scrape_recipe_ldata(url: str) -> dict:
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-        "Accept-Language": "en-US,en;q=0.9",
-        "Referer": "https://www.google.com/",
-    }
-
     # Load the requested URL with headers
     async with httpx.AsyncClient() as client:
-        response = await client.get(url, headers=headers, follow_redirects=True)
-        response.raise_for_status()
+        response = await client.get(url, headers=HEADERS, follow_redirects=True)
+        if response.status_code >= 300:
+            return None
 
         # Extract the recipe ld+json data
         soup = BeautifulSoup(response.text, 'html.parser')