jacob/munch-ease-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Catch a broader exception type in scrapers

Browse source
This commit is contained in:
jableader 2024-10-28 17:14:16 +11:00
parent 05d6cd9c01
commit b17bb1d687
2 changed files with 8 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
products/coles.py
View file

@ -54,7 +54,7 @@ async def _request_details(product_id: str) -> dict:
response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
response.raise_for_status()
return response.json()
except httpx.NetworkError as ne:
except httpx.HTTPError as ne:
print(ne)
api_details = None

16
products/woolworths.py
View file

@ -36,24 +36,22 @@ def _get_package_size(data: dict) -> str:
def _get_client() -> httpx.AsyncClient:
return httpx.AsyncClient()
last_cookies = None
cached_cookies = None
async def _request_url(url: str) -> dict:
global last_cookies
global cached_cookies
async with _get_client() as client:
if last_cookies:
cookies = last_cookies
else:
cookies = await _get_cookies(client)
last_cookies = cookies
if not cached_cookies:
cached_cookies = await _get_cookies(client)
cookies = cached_cookies
try:
response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
response.raise_for_status()
return response.json()
except httpx.NetworkError as ne:
except httpx.HTTPError as ne:
print(ne)
last_cookies = None
cached_cookies = None
return None
def _get_product_details_url(product_id) -> str: