jacob/munch-ease-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Catch a broader exception type in scrapers

Browse source
This commit is contained in:
jableader 2024-10-28 17:14:16 +11:00
parent 05d6cd9c01
commit b17bb1d687
2 changed files with 8 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
products/coles.py
View file

@ -54,7 +54,7 @@ async def _request_details(product_id: str) -> dict:
response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies) response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
except httpx.NetworkError as ne: except httpx.HTTPError as ne:
print(ne) print(ne)
api_details = None api_details = None

16
products/woolworths.py
View file

@ -36,24 +36,22 @@ def _get_package_size(data: dict) -> str:
def _get_client() -> httpx.AsyncClient: def _get_client() -> httpx.AsyncClient:
return httpx.AsyncClient() return httpx.AsyncClient()
last_cookies = None cached_cookies = None
async def _request_url(url: str) -> dict: async def _request_url(url: str) -> dict:
global last_cookies global cached_cookies
async with _get_client() as client: async with _get_client() as client:
if last_cookies: if not cached_cookies:
cookies = last_cookies cached_cookies = await _get_cookies(client)
else:
cookies = await _get_cookies(client)
last_cookies = cookies
cookies = cached_cookies
try: try:
response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies) response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
except httpx.NetworkError as ne: except httpx.HTTPError as ne:
print(ne) print(ne)
last_cookies = None cached_cookies = None
return None return None
def _get_product_details_url(product_id) -> str: def _get_product_details_url(product_id) -> str: