munch-ease-backend/products/woolworths.py

import re
from typing import Optional, Tuple

import httpx

HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Accept-Encoding": "gzip, deflate, br",
    "DNT": "1",
    "Sec-GPC": "1",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Pragma": "no-cache",
    "Cache-Control": "no-cache",
}


async def _get_cookies(client):
    # Make a request to https://www.woolworths.com.au/ as if we were a normal browser, then return the cookies
    response = await client.get(
        "https://www.woolworths.com.au/", headers=HEADERS, follow_redirects=True
    )
    return dict(response.cookies)


def _get_package_size(data: dict) -> Tuple[int, str]:
    size = data["Product"]["PackageSize"]
    if size:
        match = re.match(r"(\d+)(.*)", size)
        if match:
            return int(match.group(1)), match.group(2)

    return 1, "items"


def _get_client() -> httpx.AsyncClient:
    return httpx.AsyncClient()


cached_cookies = None


async def _request_url(url: str) -> Optional[dict]:
    global cached_cookies

    async with _get_client() as client:
        if not cached_cookies:
            cached_cookies = await _get_cookies(client)

        cookies = cached_cookies
        try:
            response = await client.get(
                url, headers=HEADERS, follow_redirects=True, cookies=cookies
            )
            response.raise_for_status()
            return response.json()
        except httpx.HTTPError as ne:
            print(ne)
            cached_cookies = None
            return None


def _get_product_details_url(product_id: str) -> str:
    return f"https://www.woolworths.com.au/apis/ui/product/detail/{product_id}"


def get_product_id(url: str) -> Optional[str]:
    woolies_regex = r"https://www.woolworths.com.au/shop/productdetails/(\d+)/?.*"
    match = re.match(woolies_regex, url)
    if match:
        return match.group(1)
    return None


async def scrape(product_id: str) -> Tuple[dict, dict]:
    details_url = _get_product_details_url(product_id)
    raw_data = await _request_url(details_url)
    if raw_data is None:
        # Return a minimal structure; callers treat this as raw payload for logging
        raw_data = {}
    quantity, unit = _get_package_size(raw_data)

    product_data = {
        "name": raw_data["Product"]["Name"],
        "quantity": quantity,
        "unit": unit,
        "img_small": raw_data["Product"]["SmallImageFile"],
        "img_large": raw_data["Product"]["LargeImageFile"],
    }

    return product_data, raw_data