From 6d7a8fb7b84d7c389c68c48b62b29a94da266625 Mon Sep 17 00:00:00 2001 From: jableader Date: Sun, 19 May 2024 22:35:37 +1000 Subject: [PATCH] Parse quantity from product --- products/__init__.py | 29 +++++++++++++++++++++++++++++ products/db.py | 19 ++++++++++++++----- tests/test_data.py | 16 ++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/products/__init__.py b/products/__init__.py index e36e03c..86da4bb 100644 --- a/products/__init__.py +++ b/products/__init__.py @@ -4,6 +4,16 @@ from products.db import Product, find_product_by_tag, find_product_by_product_id from products.scraping import scrape_woolies_data, get_product_id, get_product_details_url from typing import List +import re + +def get_package_size(data: dict) -> str: + size = data['Product']['PackageSize'] + if size: + match = re.match(r'(\d+)(.*)', size) + if match: + return int(match.group(1)), match.group(2) + + return 1, 'items' async def create_product(link: str) -> Product: product_id = get_product_id(link) @@ -13,11 +23,16 @@ async def create_product(link: str) -> Product: product_url = get_product_details_url(product_id) product, data = None, await scrape_woolies_data(product_url) if data: + _dump_json_data_to_log(data, product_id) + + quantity, unit = get_package_size(data) product = Product( id=0, product_id=product_id, name=data['Product']['Name'], link=link, + quantity=quantity, + unit=unit, img_small=data['Product']['SmallImageFile'], img_large=data['Product']['LargeImageFile'], ) @@ -54,3 +69,17 @@ async def get_or_create(conn, url: str, tags: List[str]) -> Product: await add_missing_tags(conn, product, tags) return product + +def _dump_json_data_to_log(data: dict, product_id: str) -> str: + import os, re + dir = './data/dump' + if not os.path.exists(dir): + os.makedirs(dir) + + prefix = f'product_{product_id}' + suffix = '.json' + file_ids = [int(re.findall(r'\d+', f)[0]) for f in os.listdir(dir) if re.match(prefix + r'\d+' + suffix, f)] + id = max(file_ids) + 1 if file_ids else 0 + filename = f'{prefix}{id}{suffix}' + with open(os.path.join(dir, filename), 'w') as f: + json.dump(data, f, indent=4) \ No newline at end of file diff --git a/products/db.py b/products/db.py index 295b02a..ec7ac4f 100644 --- a/products/db.py +++ b/products/db.py @@ -4,11 +4,15 @@ from pydantic import BaseModel import json class Product(BaseModel): - KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large'] + KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'quantity', 'unit', 'img_small', 'img_large'] + NON_INSERT_KEYS: ClassVar[List[str]] = ['id'] + id: int product_id: str link: str name: str + quantity: int + unit: str img_small: str img_large: str @@ -19,6 +23,8 @@ async def create(conn): product_id TEXT UNIQUE, link TEXT, name TEXT, + quantity INTEGER, + unit TEXT, img_small TEXT, img_large TEXT, raw_data TEXT @@ -62,10 +68,13 @@ async def find_product_by_product_id(conn, product_id: str) -> Product: return Product(**{k:v for k,v in zip(Product.KEYS, row)}) async def insert_product(conn, product: Product, data: dict): - async with conn.execute(''' - INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data) - VALUES (?, ?, ?, ?, ?, ?) - ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, json.dumps(data))) as cursor: + insert_keys = [k for k in Product.KEYS if k not in Product.NON_INSERT_KEYS] + insert_values = [getattr(product, k) for k in insert_keys] + + async with conn.execute(f''' + INSERT INTO Product ({','.join(insert_keys)}, raw_data) + VALUES ({','.join(['?'] * len(insert_keys))}, ?) + ''', (*insert_values, json.dumps(data))) as cursor: product.id = cursor.lastrowid await conn.commit() diff --git a/tests/test_data.py b/tests/test_data.py index 29e11cf..459be17 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -24,6 +24,8 @@ class Products: id=0, name="Fresh Broccoli", product_id="134681", + quantity=1, + unit="Items", link="https://www.woolworths.com.au/shop/productdetails/134681/fresh-broccoli", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/134681.jpg", img_large="https://cdn0.woolworths.media/content/wowproductimages/large/134681.jpg", @@ -34,6 +36,8 @@ class Products: id=0, name="La Famiglia Garlic Bread", product_id="294517", + quantity=1, + unit="Loaf", link="https://www.woolworths.com.au/shop/productdetails/294517/la-famiglia-garlic-bread", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/294517.jpg", img_large="https://cdn0.woolworths.media/content/wowproductimages/large/294517.jpg", @@ -44,6 +48,8 @@ class Products: id=0, name="Beans Round", product_id="134072", + quantity=1, + unit="kg", link="https://www.woolworths.com.au/shop/productdetails/134072/beans-round", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/134072.jpg", img_large="https://cdn0.woolworths.media/content/wowproductimages/large/134072.jpg", @@ -54,6 +60,8 @@ class Products: id=0, name="Western Star Unsalted Butter Chef's Choice", product_id="712251", + quantity=500, + unit="g", link="https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg", img_large="https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg", @@ -63,6 +71,8 @@ class Products: saxa_iodised_table_salt_shaker = products.Product( id=0, name="Saxa Iodised Table Salt Shaker", + quantity=750, + unit="g", product_id="33245", link="https://www.woolworths.com.au/shop/productdetails/33245/saxa-iodised-table-salt-shaker", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/033245.jpg", @@ -73,6 +83,8 @@ class Products: mckenzies_pepper_black_ground = products.Product( id=0, name="Mckenzie's Pepper Black Ground", + quantity=100, + unit="g", product_id="75194", link="https://www.woolworths.com.au/shop/productdetails/75194/mckenzie-s-pepper-black-ground", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/075194.jpg", @@ -84,6 +96,8 @@ class Products: id=0, name="Apple", product_id="3542", + quantity=1, + unit="Items", link="https://www.woolworths.com.au/shop/productdetails/0/apple", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/0.jpg", img_large="https://cdn0.woolworths.media/content/wowproductimages/large/0.jpg", @@ -94,6 +108,8 @@ class Products: id=0, name="Banana", product_id="214", + quantity=1, + unit="Items", link="https://www.woolworths.com.au/shop/productdetails/0/banana", img_small="https://cdn0.woolworths.media/content/wowproductimages/small/0.jpg", img_large="https://cdn0.woolworths.media/content/wowproductimages/large/0.jpg",