From 0fdd0d10c24fbbdb05077fff6498e7c15aa63f02 Mon Sep 17 00:00:00 2001
From: jableader <jacobdunk@gmail.com>
Date: Tue, 1 Oct 2024 20:02:32 +1000
Subject: [PATCH] improved scraper

---
 recipes/scraping.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/recipes/scraping.py b/recipes/scraping.py
index 38fe5cb..3fd2cd8 100644
--- a/recipes/scraping.py
+++ b/recipes/scraping.py
@@ -2,6 +2,22 @@ from bs4 import BeautifulSoup
 import httpx
 import json
 
+HEADERS = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'DNT': '1',
+    'Sec-GPC': '1',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Priority': 'u=1',
+    'Pragma': 'no-cache',
+    'Cache-Control': 'no-cache',
+}
+
 def _is_recipe_ldata(ldata_node):
     if '@type' in ldata_node:
         typ = ldata_node['@type']
@@ -14,16 +30,11 @@ def _is_recipe_ldata(ldata_node):
     return None
 
 async def scrape_recipe_ldata(url: str) -> dict:
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-        "Accept-Language": "en-US,en;q=0.9",
-        "Referer": "https://www.google.com/",
-    }
-
     # Load the requested URL with headers
     async with httpx.AsyncClient() as client:
-        response = await client.get(url, headers=headers, follow_redirects=True)
-        response.raise_for_status()
+        response = await client.get(url, headers=HEADERS, follow_redirects=True)
+        if response.status_code >= 300:
+            return None
 
         # Extract the recipe ld+json data
         soup = BeautifulSoup(response.text, 'html.parser')