55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
|
import re, httpx
|
||
|
|
|
||
|
|
HEADERS = {
|
||
|
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
||
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
||
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
||
|
|
'DNT': '1',
|
||
|
|
'Sec-GPC': '1',
|
||
|
|
'Connection': 'keep-alive',
|
||
|
|
'Upgrade-Insecure-Requests': '1',
|
||
|
|
'Sec-Fetch-Dest': 'document',
|
||
|
|
'Sec-Fetch-Mode': 'navigate',
|
||
|
|
'Sec-Fetch-Site': 'none',
|
||
|
|
'Sec-Fetch-User': '?1',
|
||
|
|
'Pragma': 'no-cache',
|
||
|
|
'Cache-Control': 'no-cache',
|
||
|
|
}
|
||
|
|
|
||
|
|
async def _get_cookies(client):
|
||
|
|
# Make a request to https://www.woolworths.com.au/ as if we were a normal browser, then return the cookies
|
||
|
|
response = await client.get('https://www.woolworths.com.au/', headers=HEADERS, follow_redirects=True)
|
||
|
|
return dict(response.cookies)
|
||
|
|
|
||
|
|
last_cookies = None
|
||
|
|
async def scrape_woolies_data(url: str) -> dict:
|
||
|
|
global last_cookies
|
||
|
|
|
||
|
|
async with httpx.AsyncClient() as client:
|
||
|
|
if last_cookies:
|
||
|
|
cookies = last_cookies
|
||
|
|
else:
|
||
|
|
cookies = await _get_cookies(client)
|
||
|
|
last_cookies = cookies
|
||
|
|
|
||
|
|
try:
|
||
|
|
response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
|
||
|
|
response.raise_for_status()
|
||
|
|
return response.json()
|
||
|
|
except httpx.NetworkError as ne:
|
||
|
|
print(ne)
|
||
|
|
last_cookies = None
|
||
|
|
return None
|
||
|
|
|
||
|
|
def get_product_id(url: str) -> str:
|
||
|
|
woolies_regex = r'https://www.woolworths.com.au/shop/productdetails/(\d+)/?.*'
|
||
|
|
match = re.match(woolies_regex, url)
|
||
|
|
if match:
|
||
|
|
return match.group(1)
|
||
|
|
return None
|
||
|
|
|
||
|
|
def get_product_details_url(product_id) -> str:
|
||
|
|
return f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}'
|
||
|
|
|