Parse quantity from product

products/__init__.py

@@ -4,6 +4,16 @@ from products.db import Product, find_product_by_tag, find_product_by_product_id
 from products.scraping import scrape_woolies_data, get_product_id, get_product_details_url
 
 from typing import List
+import re
+
+def get_package_size(data: dict) -> str:
+    size = data['Product']['PackageSize']
+    if size:
+        match = re.match(r'(\d+)(.*)', size)
+        if match:
+            return int(match.group(1)), match.group(2)
+
+    return 1, 'items'
 
 async def create_product(link: str) -> Product:
     product_id = get_product_id(link)
@@ -13,11 +23,16 @@ async def create_product(link: str) -> Product:
     product_url = get_product_details_url(product_id)
     product, data = None, await scrape_woolies_data(product_url)
     if data:
+        _dump_json_data_to_log(data, product_id)
+
+        quantity, unit = get_package_size(data)
         product =  Product(
             id=0,
             product_id=product_id,
             name=data['Product']['Name'],
             link=link,
+            quantity=quantity,
+            unit=unit,
             img_small=data['Product']['SmallImageFile'],
             img_large=data['Product']['LargeImageFile'],
         )
@@ -54,3 +69,17 @@ async def get_or_create(conn, url: str, tags: List[str]) -> Product:
         await add_missing_tags(conn, product, tags)
 
     return product
+
+def _dump_json_data_to_log(data: dict, product_id: str) -> str:
+    import os, re
+    dir = './data/dump'
+    if not os.path.exists(dir):
+        os.makedirs(dir)
+
+    prefix = f'product_{product_id}'
+    suffix = '.json'
+    file_ids = [int(re.findall(r'\d+', f)[0]) for f in os.listdir(dir) if re.match(prefix + r'\d+' + suffix, f)]
+    id = max(file_ids) + 1 if file_ids else 0
+    filename = f'{prefix}{id}{suffix}'
+    with open(os.path.join(dir, filename), 'w') as f:
+        json.dump(data, f, indent=4)

products/db.py

@@ -4,11 +4,15 @@ from pydantic import BaseModel
 import json
 
 class Product(BaseModel):
-    KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large']
+    KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'quantity', 'unit', 'img_small', 'img_large']
+    NON_INSERT_KEYS: ClassVar[List[str]] = ['id']
+
     id: int
     product_id: str
     link: str
     name: str
+    quantity: int
+    unit: str
     img_small: str
     img_large: str
 
@@ -19,6 +23,8 @@ async def create(conn):
             product_id TEXT UNIQUE,
             link TEXT,
             name TEXT,
+            quantity INTEGER,
+            unit TEXT,
             img_small TEXT,
             img_large TEXT,
             raw_data TEXT
@@ -62,10 +68,13 @@ async def find_product_by_product_id(conn, product_id: str) -> Product:
             return Product(**{k:v for k,v in zip(Product.KEYS, row)})
 
 async def insert_product(conn, product: Product, data: dict):
-    async with conn.execute('''
+    insert_keys = [k for k in Product.KEYS if k not in Product.NON_INSERT_KEYS]
-        INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data)
+    insert_values = [getattr(product, k) for k in insert_keys]
-        VALUES (?, ?, ?, ?, ?, ?)
+
-    ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, json.dumps(data))) as cursor:
+    async with conn.execute(f'''
+        INSERT INTO Product ({','.join(insert_keys)}, raw_data)
+        VALUES ({','.join(['?'] * len(insert_keys))}, ?)
+    ''', (*insert_values, json.dumps(data))) as cursor:
         product.id = cursor.lastrowid
     
     await conn.commit()

tests/test_data.py

@@ -24,6 +24,8 @@ class Products:
         id=0,
         name="Fresh Broccoli",
         product_id="134681",
+        quantity=1,
+        unit="Items",
         link="https://www.woolworths.com.au/shop/productdetails/134681/fresh-broccoli",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/134681.jpg",
         img_large="https://cdn0.woolworths.media/content/wowproductimages/large/134681.jpg",
@@ -34,6 +36,8 @@ class Products:
         id=0,
         name="La Famiglia Garlic Bread",
         product_id="294517",
+        quantity=1,
+        unit="Loaf",
         link="https://www.woolworths.com.au/shop/productdetails/294517/la-famiglia-garlic-bread",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/294517.jpg",
         img_large="https://cdn0.woolworths.media/content/wowproductimages/large/294517.jpg",
@@ -44,6 +48,8 @@ class Products:
         id=0,
         name="Beans Round",
         product_id="134072",
+        quantity=1,
+        unit="kg",
         link="https://www.woolworths.com.au/shop/productdetails/134072/beans-round",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/134072.jpg",
         img_large="https://cdn0.woolworths.media/content/wowproductimages/large/134072.jpg",
@@ -54,6 +60,8 @@ class Products:
         id=0,
         name="Western Star Unsalted Butter Chef's Choice",
         product_id="712251",
+        quantity=500,
+        unit="g",
         link="https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
         img_large="https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg",
@@ -63,6 +71,8 @@ class Products:
     saxa_iodised_table_salt_shaker = products.Product(
         id=0,
         name="Saxa Iodised Table Salt Shaker",
+        quantity=750,
+        unit="g",
         product_id="33245",
         link="https://www.woolworths.com.au/shop/productdetails/33245/saxa-iodised-table-salt-shaker",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/033245.jpg",
@@ -73,6 +83,8 @@ class Products:
     mckenzies_pepper_black_ground = products.Product(
         id=0,
         name="Mckenzie's Pepper Black Ground",
+        quantity=100,
+        unit="g",
         product_id="75194",
         link="https://www.woolworths.com.au/shop/productdetails/75194/mckenzie-s-pepper-black-ground",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/075194.jpg",
@@ -84,6 +96,8 @@ class Products:
         id=0,
         name="Apple",
         product_id="3542",
+        quantity=1,
+        unit="Items",
         link="https://www.woolworths.com.au/shop/productdetails/0/apple",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/0.jpg",
         img_large="https://cdn0.woolworths.media/content/wowproductimages/large/0.jpg",
@@ -94,6 +108,8 @@ class Products:
         id=0,
         name="Banana",
         product_id="214",
+        quantity=1,
+        unit="Items",
         link="https://www.woolworths.com.au/shop/productdetails/0/banana",
         img_small="https://cdn0.woolworths.media/content/wowproductimages/small/0.jpg",
         img_large="https://cdn0.woolworths.media/content/wowproductimages/large/0.jpg",