jacob/munch-ease-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Coles dynamic get api version with cookies

Browse source
This commit is contained in:
jableader 2024-10-24 22:14:05 +11:00
parent a0c2b88a55
commit 05d6cd9c01
1 changed files with 21 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
products/coles.py
View file

@ -20,8 +20,13 @@ HEADERS = {
async def _get_cookies(client):
# Make a request to https://www.coles.com.au/ as if we were a normal browser, then return the cookies
response = await client.get('https://www.coles.com.au/', headers=HEADERS, follow_redirects=True)
return dict(response.cookies)
response = await client.get('https://www.coles.com.au/product/coles-strawberries-250g-5191256', headers=HEADERS, follow_redirects=True)
version = re.findall(r'202[4-9][01]\d[0-2]\d.02_v\d.\d\d.\d', response.text)
if len(version) == 0:
raise Exception('Could not find the Coles API')
return dict(response.cookies), version[0]
def _get_package_size(size: str) -> Union[int, str]:
if size:
@ -34,30 +39,32 @@ def _get_package_size(size: str) -> Union[int, str]:
def _get_client():
return httpx.AsyncClient()
last_cookies = None
async def _request_url(url: str) -> dict:
global last_cookies
api_details = None
async def _request_details(product_id: str) -> dict:
global api_details
async with _get_client() as client:
if last_cookies:
cookies = last_cookies
else:
cookies = await _get_cookies(client)
last_cookies = cookies
if not api_details:
api_details = await _get_cookies(client)
cookies, api_version = api_details
url = _get_product_details_url(api_version, product_id)
try:
response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
response.raise_for_status()
return response.json()
except httpx.NetworkError as ne:
print(ne)
last_cookies = None
api_details = None
return None
def _get_product_details_url(product_id) -> str:
def _get_product_details_url(version: str, product_id: str) -> str:
# https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/cadbury-favourites-boxed-chocolate-340g-3571992.json?slug=cadbury-favourites-boxed-chocolate-340g-3571992
# https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/coles-blueberries-170g-3571948.json?slug=coles-blueberries-170g-3571948
return f'https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/{product_id}.json'
#
return f'https://www.coles.com.au/_next/data/{version}/en/product/{product_id}.json'
def get_product_id(url: str) -> str:
# https://www.coles.com.au/product/cadbury-favourites-boxed-chocolate-340g-3571992
@ -68,8 +75,7 @@ def get_product_id(url: str) -> str:
return None
async def scrape(product_id: str) -> Union[dict, dict]:
details_url = _get_product_details_url(product_id)
raw_data = await _request_url(details_url)
raw_data = await _request_details(product_id)
product = raw_data['pageProps']['product']