Add support for coles, and tests

products/__init__.py

@@ -1,44 +1,20 @@
 import json
 
-from products.db import Product, find_product_by_tag, find_product_by_product_id, insert_product, get_tags, add_tag, find_product_by_id
-from products.scraping import scrape_woolies_data, get_product_id, get_product_details_url
+from products.db import Product, find_product_by_tag, find_product_by_key, insert_product, get_tags, add_tag, find_product_by_id
 
-from typing import List
+from products import woolworths, coles
+SCRAPERS = { 'woolworths': woolworths, 'coles': coles }
+
+from typing import List, Union
 import re
 
-def get_package_size(data: dict) -> str:
-    size = data['Product']['PackageSize']
-    if size:
-        match = re.match(r'(\d+)(.*)', size)
-        if match:
-            return int(match.group(1)), match.group(2)
+def _get_shop_key(link: str) -> Union[str, str]: # (shop_code, product_id)
+    for shop_code, shop_scraper in SCRAPERS.items():
+        product_id = shop_scraper.get_product_id(link)
+        if product_id:
+            return shop_code, product_id
+    return None, None
 
-    return 1, 'items'
-
-async def create_product(link: str) -> Product:
-    product_id = get_product_id(link)
-    if not product_id:
-        return None, None
-    
-    product_url = get_product_details_url(product_id)
-    product, data = None, await scrape_woolies_data(product_url)
-    if data:
-        _dump_json_data_to_log(data, product_id)
-
-        quantity, unit = get_package_size(data)
-        product = Product(
-            id=0,
-            product_id=product_id,
-            name=data['Product']['Name'],
-            link=link,
-            quantity=quantity,
-            unit=unit,
-            img_small=data['Product']['SmallImageFile'],
-            img_large=data['Product']['LargeImageFile'],
-        )
-
-    return product, data
-    
 async def add_missing_tags(conn, product: Product, tags: List[str]):
     existing_tags = set()
     async for tag in get_tags(conn, product):
@@ -54,19 +30,26 @@ async def add_missing_tags(conn, product: Product, tags: List[str]):
     return product
 
 async def get_or_create(conn, url: str, tags: List[str]) -> Product:
-    product_id = get_product_id(url)
+    shop_code, product_id = _get_shop_key(url)
     if not product_id:
         return None
     
-    existing = await find_product_by_product_id(conn, product_id)
+    existing = await find_product_by_key(conn, shop_code, product_id)
     if existing:
         await add_missing_tags(conn, existing, tags)
         return existing
     
-    product, data = await create_product(url)
-    if product:
-        await insert_product(conn, product, data)
-        await add_missing_tags(conn, product, tags)
+    product_data, raw_response = await SCRAPERS[shop_code].scrape(product_id)
+    product = Product(
+        id=-1,
+        shop_code=shop_code,
+        product_id=product_id,
+        link=url,
+        **product_data
+    )
+
+    await insert_product(conn, product, raw_response)
+    await add_missing_tags(conn, product, tags)
 
     return product
 

products/coles.py

@@ -0,0 +1,88 @@
+import re, httpx
+from typing import Union
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'DNT': '1',
+    'Sec-GPC': '1',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Pragma': 'no-cache',
+    'Cache-Control': 'no-cache',
+}
+
+async def _get_cookies(client):
+    # Make a request to https://www.coles.com.au/ as if we were a normal browser, then return the cookies
+    response = await client.get('https://www.coles.com.au/', headers=HEADERS, follow_redirects=True)
+    return dict(response.cookies)
+
+def _get_package_size(size: str) -> Union[int, str]:
+    if size:
+        match = re.match(r'(\d+)(.*)', size)
+        if match:
+            return int(match.group(1)), match.group(2)
+
+    return 1, 'items'
+
+def _get_client():
+    return httpx.AsyncClient()
+
+last_cookies = None
+async def _request_url(url: str) -> dict:
+    global last_cookies
+
+    async with _get_client() as client:
+        if last_cookies:
+            cookies = last_cookies
+        else:
+            cookies = await _get_cookies(client)
+            last_cookies = cookies
+
+        try:
+            response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
+            response.raise_for_status()
+            return response.json()
+        except httpx.NetworkError as ne:
+            print(ne)
+            last_cookies = None
+            return None
+
+def _get_product_details_url(product_id) -> str:
+    # https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/cadbury-favourites-boxed-chocolate-340g-3571992.json?slug=cadbury-favourites-boxed-chocolate-340g-3571992
+    # https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/coles-blueberries-170g-3571948.json?slug=coles-blueberries-170g-3571948
+    return f'https://www.coles.com.au/_next/data/20240926.02_v4.18.0/en/product/{product_id}.json'
+
+def get_product_id(url: str) -> str:
+    # https://www.coles.com.au/product/cadbury-favourites-boxed-chocolate-340g-3571992
+    regex = r'https://www.coles.com.au/product/([^/]+)/?.*'
+    match = re.match(regex, url)
+    if match:
+        return match.group(1)
+    return None
+
+async def scrape(product_id: str) -> Union[dict, dict]:
+    details_url = _get_product_details_url(product_id)
+    raw_data = await _request_url(details_url)
+    
+    product = raw_data['pageProps']['product']
+
+    quantity, unit = _get_package_size(product['size'])
+
+    img_prefix = 'https://shop.coles.com.au'
+    images = product['images'][0]
+    product_data = {
+        'name': product['name'], 
+        'quantity': quantity, 
+        'unit': unit, 
+        'img_small': (img_prefix + images['thumb']['path']) if images else None,
+        'img_large': (img_prefix + images['full']['path']) if images else None,
+    }
+
+    return product_data, raw_data

products/db.py

@@ -4,11 +4,12 @@ from pydantic import BaseModel
 import json
 
 class Product(BaseModel):
-    KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'quantity', 'unit', 'img_small', 'img_large']
+    KEYS: ClassVar[List[str]] = ['id', 'product_id', 'shop_code', 'link', 'name', 'quantity', 'unit', 'img_small', 'img_large']
     NON_INSERT_KEYS: ClassVar[List[str]] = ['id']
 
     id: int = -1
     product_id: str
+    shop_code: str
     link: str
     name: str
     quantity: int
@@ -20,11 +21,12 @@ async def create(conn):
     await conn.execute('''
         CREATE TABLE IF NOT EXISTS Product (
             id INTEGER PRIMARY KEY,
-            product_id TEXT UNIQUE,
-            link TEXT,
-            name TEXT,
-            quantity INTEGER,
-            unit TEXT,
+            product_id TEXT UNIQUE NOT NULL,
+            shop_code TEXT NOT NULL,
+            link TEXT NOT NULL,
+            name TEXT NOT NULL,
+            quantity INTEGER NOT NULL,
+            unit TEXT NOT NULL,
             img_small TEXT,
             img_large TEXT,
             raw_data TEXT
@@ -58,12 +60,12 @@ async def find_product_by_id(conn, product_id: str) -> Product:
         async for row in cursor:
             return Product(**{k:v for k,v in zip(Product.KEYS, row)})
 
-async def find_product_by_product_id(conn, product_id: str) -> Product:
+async def find_product_by_key(conn, shop_code: str, product_id: str) -> Product:
     async with conn.execute(f'''
         SELECT {','.join(Product.KEYS)} FROM Product
-        WHERE product_id = ?
+        WHERE shop_code = ? AND product_id = ?
         LIMIT 1
-    ''', (product_id,)) as cursor:
+    ''', (shop_code, product_id,)) as cursor:
         async for row in cursor:
             return Product(**{k:v for k,v in zip(Product.KEYS, row)})
 

products/woolworths.py

@@ -1,5 +1,7 @@
 import re, httpx
 
+from typing import Union
+
 HEADERS = {
     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
@@ -22,11 +24,23 @@ async def _get_cookies(client):
     response = await client.get('https://www.woolworths.com.au/', headers=HEADERS, follow_redirects=True)
     return dict(response.cookies)
 
+def _get_package_size(data: dict) -> str:
+    size = data['Product']['PackageSize']
+    if size:
+        match = re.match(r'(\d+)(.*)', size)
+        if match:
+            return int(match.group(1)), match.group(2)
+
+    return 1, 'items'
+
+def _get_client() -> httpx.AsyncClient:
+    return httpx.AsyncClient()
+
 last_cookies = None
-async def scrape_woolies_data(url: str) -> dict:
+async def _request_url(url: str) -> dict:
     global last_cookies
 
-    async with httpx.AsyncClient() as client:
+    async with _get_client() as client:
         if last_cookies:
             cookies = last_cookies
         else:
@@ -42,6 +56,9 @@ async def scrape_woolies_data(url: str) -> dict:
             last_cookies = None
             return None
 
+def _get_product_details_url(product_id) -> str:
+    return f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}'
+
 def get_product_id(url: str) -> str:
     woolies_regex = r'https://www.woolworths.com.au/shop/productdetails/(\d+)/?.*'
     match = re.match(woolies_regex, url)
@@ -49,6 +66,17 @@ def get_product_id(url: str) -> str:
         return match.group(1)
     return None
 
-def get_product_details_url(product_id) -> str:
-    return f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}'
+async def scrape(product_id: str) -> Union[dict, dict]:
+    details_url = _get_product_details_url(product_id)
+    raw_data = await _request_url(details_url)
+    quantity, unit = _get_package_size(raw_data)
 
+    product_data = {
+        'name': raw_data['Product']['Name'],
+        'quantity': quantity,
+        'unit': unit,
+        'img_small': raw_data['Product']['SmallImageFile'],
+        'img_large': raw_data['Product']['LargeImageFile'],
+    }
+
+    return product_data, raw_data

tests/httpx_mocks.py

@@ -0,0 +1,124 @@
+import httpx
+import json
+import os
+
+class RecordingAsyncClient:
+    def __init__(self, save_dir: str):
+        self.save_dir = save_dir
+        os.makedirs(self.save_dir, exist_ok=True)
+        self.client = None  # Will be initialized in __aenter__
+        
+    async def __aenter__(self):
+        # Initialize the actual AsyncClient when entering the context manager
+        self.client = httpx.AsyncClient()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        # Ensure the client is closed when exiting the context manager
+        await self.client.aclose()
+    
+    async def request(self, method: str, url: str, **kwargs):
+        # Send the actual request
+        response = await self.client.request(method, url, **kwargs)
+        
+        # Record the request and response
+        record = {
+            "request": {
+                "method": method,
+                "url": url,
+                "headers": dict(response.request.headers),
+                "content": response.request.content.decode('utf-8', errors='ignore'),
+            },
+            "response": {
+                "status_code": response.status_code,
+                "headers": dict(response.headers),
+                "content": response.text,
+                "cookies": dict(response.cookies),
+            }
+        }
+
+        # Generate a filename based on the URL and method
+        record_file = os.path.join(self.save_dir, f"{method}_{url.replace('/', '_')}.json")
+        
+        # Save the record to a file
+        with open(record_file, 'w') as f:
+            json.dump(record, f, indent=4)
+
+        return response
+
+    async def get(self, url: str, **kwargs):
+        return await self.request("GET", url, **kwargs)
+    
+    async def post(self, url: str, **kwargs):
+        return await self.request("POST", url, **kwargs)
+
+    async def put(self, url: str, **kwargs):
+        return await self.request("PUT", url, **kwargs)
+
+    async def delete(self, url: str, **kwargs):
+        return await self.request("DELETE", url, **kwargs)
+
+from unittest.mock import Mock
+import os
+import json
+
+class MockAsyncClient:
+    def __init__(self, load_dir: str):
+        self.load_dir = load_dir
+    
+    async def __aenter__(self):
+        # No actual client to initialize, just return the instance
+        return self
+    
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        # No actual client to close
+        pass
+
+    async def request(self, method: str, url: str, **kwargs):
+        # Generate the filename based on the URL and method
+        record_file = os.path.join(self.load_dir, f"{method}_{url.replace('/', '_')}.json")
+        
+        if not os.path.exists(record_file):
+            raise FileNotFoundError(f"Recorded response not found for {method} {url}")
+        
+        # Load the recorded response from the file
+        with open(record_file, 'r') as f:
+            record = json.load(f)
+        
+        # Create a mock response object
+        mock_response = Mock()
+        
+        # Mock the status code
+        mock_response.status_code = record['response']['status_code']
+        
+        # Mock the json method to return the content as a parsed JSON
+        def mock_json():
+            try:
+                return json.loads(record['response']['content'])
+            except json.JSONDecodeError:
+                return record['response']['content']
+        
+        mock_response.json = mock_json
+        
+        # Mock the cookies as a dictionary
+        mock_response.cookies = record['response']['cookies']
+
+        # Mock the headers as a dictionary
+        mock_response.headers = record['response']['headers']
+        
+        # Mock the text attribute
+        mock_response.text = record['response']['content']
+        
+        return mock_response
+
+    async def get(self, url: str, **kwargs):
+        return await self.request("GET", url, **kwargs)
+    
+    async def post(self, url: str, **kwargs):
+        return await self.request("POST", url, **kwargs)
+
+    async def put(self, url: str, **kwargs):
+        return await self.request("PUT", url, **kwargs)
+
+    async def delete(self, url: str, **kwargs):
+        return await self.request("DELETE", url, **kwargs)

tests/sample_files/coles/GET_https:__www.coles.com.au_.json

@@ -0,0 +1,41 @@
+{
+    "request": {
+        "method": "GET",
+        "url": "https://www.coles.com.au/",
+        "headers": {
+            "host": "www.coles.com.au",
+            "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0",
+            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "accept-language": "en-US,en;q=0.5",
+            "accept-encoding": "gzip, deflate, br",
+            "dnt": "1",
+            "sec-gpc": "1",
+            "connection": "keep-alive",
+            "upgrade-insecure-requests": "1",
+            "sec-fetch-dest": "document",
+            "sec-fetch-mode": "navigate",
+            "sec-fetch-site": "none",
+            "sec-fetch-user": "?1",
+            "pragma": "no-cache",
+            "cache-control": "no-cache"
+        },
+        "content": ""
+    },
+    "response": {
+        "status_code": 200,
+        "headers": {
+            "content-type": "text/html",
+            "cache-control": "no-cache, no-store",
+            "connection": "close",
+            "content-length": "3345",
+            "x-iinfo": "7-26769261-0 0CNN RT(1727584618151 22) q(0 -1 -1 1) r(0 -1) B10(14,0,0)",
+            "strict-transport-security": "max-age=31536000; includeSubDomains",
+            "set-cookie": "visid_incap_2800108=vNYK15gzRoOHbzyO31mNh2rZ+GYAAAAAQUIPAAAAAABwIQoipD3Nw8q38f6JHIb5; expires=Sun, 28 Sep 2025 12:16:28 GMT; HttpOnly; path=/; Domain=.coles.com.au; Secure; SameSite=None, incap_ses_808_2800108=AWejGplhXmNxgGAG25c2C2rZ+GYAAAAAr91ZD4bl3op+nNtxHCpveg==; path=/; Domain=.coles.com.au; Secure; SameSite=None"
+        },
+        "content": "<!DOCTYPE html>\r\n<html>\r\n    <head>\r\n        <noscript>\r\n            <title>Pardon Our Interruption</title>\r\n        </noscript>\r\n\r\n        <meta name=\"viewport\" content=\"width=1000\">\r\n        <meta name=\"robots\" content=\"noindex, nofollow\">\r\n        <meta http-equiv=\"cache-control\" content=\"no-cache, no-store, must-revalidate\">\r\n        <meta http-equiv=\"pragma\" content=\"no-cache\">\r\n        <meta http-equiv=\"expires\" content=\"0\">\r\n\r\n        <style>\r\n            .container { max-width: 800px; margin: auto; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; color: #7a838c; }\r\n            h1 { color: #2a2d30; font-weight: 500; }\r\n            li { margin: 0 0 10px; }\r\n            a { color: #428bca; }\r\n            a:hover, a:focus { color: #2a6496; }\r\n        </style>\r\n\r\n        <script>\r\n          var isSpa = new URLSearchParams(window.location.search).get('X-SPA') === '1' || window.isImpervaSpaSupport;\r\n        </script>\r\n\r\n        <!-- This head template should be placed before the following script tag that loads the challenge script -->\r\n        <script>\r\n          window.onProtectionInitialized = function(protection) {\r\n            if (protection && protection.cookieIsSet && !protection.cookieIsSet()) {\r\n              showBlockPage();\r\n              return;\r\n            }\r\n            if (!isSpa) {\r\n              window.location.reload(true);\r\n            }\r\n          };\r\n          window.reeseSkipExpirationCheck = true;\r\n        </script>\r\n\r\n        <script>\r\n          if (!isSpa) {\r\n            var scriptElement = document.createElement('script');\r\n            scriptElement.type = \"text/javascript\";\r\n            scriptElement.src = \"/Gone-our-Graught-but-your-did-on-bounds-Do-care-/10641756131677315109?s=TcCR5Gmz\";\r\n            scriptElement.async = true;\r\n            scriptElement.defer = true;\r\n            document.head.appendChild(scriptElement);\r\n          }\r\n        </script>\r\n        \r\n    </head>\r\n    <body>\r\n\r\n        \r\n\r\n        <div class=\"container\">\r\n            <script>document.getElementsByClassName(\"container\")[0].style.display = \"none\";</script>\r\n            \r\n            <h1>Pardon Our Interruption</h1>\r\n<p>As you were browsing something about your browser made us think you were a bot. There are a few reasons this might happen:</p>\r\n<ul>\r\n<noscript><li>You've disabled JavaScript in your web browser.</li></noscript>\r\n<li>You're a power user moving through this website with super-human speed.</li>\r\n<li>You've disabled cookies in your web browser.</li>\r\n<li>A third-party browser plugin, such as Ghostery or NoScript, is preventing JavaScript from running. Additional information is available in this <a title='Third party browser plugins that block javascript' href='http://ds.tl/help-third-party-plugins' target='_blank'>support article</a>.</li>\r\n</ul>\r\n<p>To regain access, please make sure that cookies and JavaScript are enabled before reloading the page.</p>\r\n\r\n\r\n        </div>\r\n        <script>\r\n          function showBlockPage() {\r\n            document.title = \"Pardon Our Interruption\";\r\n            document.getElementsByClassName(\"container\")[0].style.display = \"block\";\r\n          }\r\n\r\n          if (isSpa) {\r\n            showBlockPage();\r\n          } else {\r\n            setTimeout(showBlockPage, 10000);\r\n          }\r\n        </script>\r\n    </body>\r\n</html>\r\n",
+        "cookies": {
+            "visid_incap_2800108": "vNYK15gzRoOHbzyO31mNh2rZ+GYAAAAAQUIPAAAAAABwIQoipD3Nw8q38f6JHIb5",
+            "incap_ses_808_2800108": "AWejGplhXmNxgGAG25c2C2rZ+GYAAAAAr91ZD4bl3op+nNtxHCpveg=="
+        }
+    }
+}

tests/test_data.py

@@ -22,6 +22,7 @@ import products
 class Products:
     broccoli = products.Product(
         id=0,
+        shop_code='woolworths',
         name="Fresh Broccoli",
         product_id="134681",
         quantity=1,
@@ -34,6 +35,7 @@ class Products:
 
     garlic_bread = products.Product(
         id=0,
+        shop_code='woolworths',
         name="La Famiglia Garlic Bread",
         product_id="294517",
         quantity=1,
@@ -46,6 +48,7 @@ class Products:
 
     beans_round = products.Product(
         id=0,
+        shop_code='woolworths',
         name="Beans Round",
         product_id="134072",
         quantity=1,
@@ -58,6 +61,7 @@ class Products:
 
     western_star_unsalted_butter_chefs_choice = products.Product(
         id=0,
+        shop_code='woolworths',
         name="Western Star Unsalted Butter Chef's Choice",
         product_id="712251",
         quantity=500,
@@ -70,6 +74,7 @@ class Products:
 
     saxa_iodised_table_salt_shaker = products.Product(
         id=0,
+        shop_code='woolworths',
         name="Saxa Iodised Table Salt Shaker",
         quantity=750,
         unit="g",
@@ -82,6 +87,7 @@ class Products:
 
     mckenzies_pepper_black_ground = products.Product(
         id=0,
+        shop_code='woolworths',
         name="Mckenzie's Pepper Black Ground",
         quantity=100,
         unit="g",
@@ -94,6 +100,7 @@ class Products:
 
     apple = products.Product(
         id=0,
+        shop_code='woolworths',
         name="Apple",
         product_id="3542",
         quantity=1,
@@ -106,6 +113,7 @@ class Products:
 
     banana = products.Product(
         id=0,
+        shop_code='woolworths',
         name="Banana",
         product_id="214",
         quantity=1,

tests/test_main.py

@@ -10,7 +10,6 @@ def reload_test_data():
 
 from db import connect, create
 
-import products.db as products_db
 import recipes.db as recipes_db
 
 def unique(lst: list, key: callable):
@@ -21,31 +20,6 @@ def unique(lst: list, key: callable):
             seen.add(k)
             yield item
 
-class TestProducts(unittest.IsolatedAsyncioTestCase):
-    async def asyncSetUp(self):
-        self.conn = await connect(':memory:')
-        await create(self.conn)
-        reload_test_data()
-        return await super().asyncSetUp()
-
-    async def asyncTearDown(self) -> None:
-        await self.conn.close()
-        return await super().asyncTearDown()
-    
-    async def testCreateAndFind(self) -> None:
-        product = test_data.Products.broccoli
-        await products_db.insert_product(self.conn, product, {})
-        self.assertIsNotNone(product)
-        self.assertGreater(product.id, 0)
-        
-        product_by_id = await products_db.find_product_by_id(self.conn, product.id)
-        self.assertIsNotNone(product_by_id)
-        self.assertEqual(product_by_id.id, product.id)
-        self.assertEqual(product_by_id.name, product.name)
-        self.assertEqual(product_by_id.link, product.link)
-        self.assertEqual(product_by_id.img_large, product.img_large)
-        self.assertEqual(product_by_id.img_small, product.img_small)
-
 import main
 
 class TestRecipe(unittest.IsolatedAsyncioTestCase):
@@ -110,6 +84,7 @@ class TestRecipe(unittest.IsolatedAsyncioTestCase):
         self.assertIsInstance(all_recipes, list, msg=all_recipes.body if hasattr(all_recipes, 'body') else all_recipes)
         self.assertEqual(len(all_recipes), 0)
 
+import products.db as products_db
 import meals
 
 class TestMeals(unittest.IsolatedAsyncioTestCase):

tests/test_products.py

@@ -0,0 +1,104 @@
+import unittest
+
+import tests.test_data as test_data
+
+import products.db as products_db
+from db import connect, create
+
+import importlib
+def reload_test_data():
+    global test_data
+    test_data = importlib.reload(test_data)
+
+class TestProductsDb(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self):
+        self.conn = await connect(':memory:')
+        await create(self.conn)
+        reload_test_data()
+        return await super().asyncSetUp()
+
+    async def asyncTearDown(self) -> None:
+        await self.conn.close()
+        return await super().asyncTearDown()
+    
+    async def testCreateAndFind(self) -> None:
+        product = test_data.Products.broccoli
+        await products_db.insert_product(self.conn, product, {})
+        self.assertIsNotNone(product)
+        self.assertGreater(product.id, 0)
+        
+        product_by_id = await products_db.find_product_by_id(self.conn, product.id)
+        self.assertIsNotNone(product_by_id)
+        self.assertEqual(product_by_id.id, product.id)
+        self.assertEqual(product_by_id.name, product.name)
+        self.assertEqual(product_by_id.link, product.link)
+        self.assertEqual(product_by_id.img_large, product.img_large)
+        self.assertEqual(product_by_id.img_small, product.img_small)
+
+
+from . import httpx_mocks
+from products import woolworths
+
+class TestWoolworths(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        local_path = './tests/sample_files/woolworths'
+        woolworths._get_client = lambda: httpx_mocks.MockAsyncClient(local_path)
+        # woolworths._get_client = lambda: httpx_mocks.RecordingAsyncClient(local_path)
+        return await super().asyncSetUp()
+    
+    async def test_get_product_id(self) -> None:
+        params = [
+            ('https://www.woolworths.com.au/shop/productdetails/144607/strawberries', '144607'),
+            ('https://www.woolworths.com.au/shop/productdetails/133211/cavendish-bananas', '133211'),
+            ('https://www.coles.com.au/product/coles-strawberries-250g-5191256', None),
+        ]
+
+        for url, id in params:
+            self.assertEqual(woolworths.get_product_id(url), id)
+
+    async def test_get_strawberries(self) -> None:
+        details, raw_data = await woolworths.scrape('144607')
+        expected = {
+            'name': 'Strawberries',
+            'quantity': 250,
+            'unit': 'g Punnet',
+            'img_small': 'https://cdn0.woolworths.media/content/wowproductimages/small/144607.jpg',
+            'img_large': 'https://cdn0.woolworths.media/content/wowproductimages/large/144607.jpg'
+        }
+
+        for key, value in expected.items():
+            self.assertEqual(details[key], value, msg=key)
+
+
+from products import coles
+
+class TestColes(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self) -> None:
+        local_path = './tests/sample_files/coles'
+        coles._get_client = lambda: httpx_mocks.MockAsyncClient(local_path)
+        # coles._get_client = lambda: httpx_mocks.RecordingAsyncClient(local_path)
+        return await super().asyncSetUp()
+    
+    async def test_get_product_id(self) -> None:
+        params = [
+            ('https://www.coles.com.au/product/coles-strawberries-250g-5191256', 'coles-strawberries-250g-5191256'),
+            ('https://www.coles.com.au/product/coles-blueberries-170g-3571948', 'coles-blueberries-170g-3571948'),
+            ('https://www.woolworths.com.au/shop/productdetails/144607/strawberries', None),
+        ]
+
+        for url, id in params:
+            self.assertEqual(coles.get_product_id(url), id)
+
+    async def test_get_strawberries(self) -> None:
+        details, raw_data = await coles.scrape('coles-strawberries-250g-5191256')
+        expected = {
+            'name': 'Strawberries',
+            'quantity': 250,
+            'unit': 'g',
+            'img_small': 'https://shop.coles.com.au/wcsstore/Coles-CAS/images/5/1/9/5191256-th.jpg',
+            'img_large': 'https://shop.coles.com.au/wcsstore/Coles-CAS/images/5/1/9/5191256.jpg'
+        }
+
+        for key, value in expected.items():
+            self.assertEqual(details[key], value, msg=key)
+