From 05d6cd9c01f22194337b3b962da2bab673e7ce3b Mon Sep 17 00:00:00 2001 From: jableader Date: Thu, 24 Oct 2024 22:14:05 +1100 Subject: [PATCH] Coles dynamic get api version with cookies --- products/coles.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/products/coles.py b/products/coles.py index 1244c7e..95bac4a 100644 --- a/products/coles.py +++ b/products/coles.py @@ -20,8 +20,13 @@ HEADERS = { async def _get_cookies(client): # Make a request to https://www.coles.com.au/ as if we were a normal browser, then return the cookies - response = await client.get('https://www.coles.com.au/', headers=HEADERS, follow_redirects=True) - return dict(response.cookies) + response = await client.get('https://www.coles.com.au/product/coles-strawberries-250g-5191256', headers=HEADERS, follow_redirects=True) + version = re.findall(r'202[4-9][01]\d[0-2]\d.02_v\d.\d\d.\d', response.text) + + if len(version) == 0: + raise Exception('Could not find the Coles API') + + return dict(response.cookies), version[0] def _get_package_size(size: str) -> Union[int, str]: if size: @@ -34,30 +39,32 @@ def _get_package_size(size: str) -> Union[int, str]: def _get_client(): return httpx.AsyncClient() -last_cookies = None -async def _request_url(url: str) -> dict: - global last_cookies +api_details = None +async def _request_details(product_id: str) -> dict: + global api_details async with _get_client() as client: - if last_cookies: - cookies = last_cookies - else: - cookies = await _get_cookies(client) - last_cookies = cookies + if not api_details: + api_details = await _get_cookies(client) + cookies, api_version = api_details + + url = _get_product_details_url(api_version, product_id) try: response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies) response.raise_for_status() return response.json() except httpx.NetworkError as ne: print(ne) - last_cookies = None + api_details = None + return None -def _get_product_details_url(product_id) -> str: +def _get_product_details_url(version: str, product_id: str) -> str: # https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/cadbury-favourites-boxed-chocolate-340g-3571992.json?slug=cadbury-favourites-boxed-chocolate-340g-3571992 # https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/coles-blueberries-170g-3571948.json?slug=coles-blueberries-170g-3571948 - return f'https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/{product_id}.json' + # + return f'https://www.coles.com.au/_next/data/{version}/en/product/{product_id}.json' def get_product_id(url: str) -> str: # https://www.coles.com.au/product/cadbury-favourites-boxed-chocolate-340g-3571992 @@ -68,8 +75,7 @@ def get_product_id(url: str) -> str: return None async def scrape(product_id: str) -> Union[dict, dict]: - details_url = _get_product_details_url(product_id) - raw_data = await _request_url(details_url) + raw_data = await _request_details(product_id) product = raw_data['pageProps']['product']