Parse quantity from product
This commit is contained in:
parent
a4639a79fd
commit
6d7a8fb7b8
3 changed files with 59 additions and 5 deletions
|
|
@ -4,6 +4,16 @@ from products.db import Product, find_product_by_tag, find_product_by_product_id
|
|||
from products.scraping import scrape_woolies_data, get_product_id, get_product_details_url
|
||||
|
||||
from typing import List
|
||||
import re
|
||||
|
||||
def get_package_size(data: dict) -> str:
|
||||
size = data['Product']['PackageSize']
|
||||
if size:
|
||||
match = re.match(r'(\d+)(.*)', size)
|
||||
if match:
|
||||
return int(match.group(1)), match.group(2)
|
||||
|
||||
return 1, 'items'
|
||||
|
||||
async def create_product(link: str) -> Product:
|
||||
product_id = get_product_id(link)
|
||||
|
|
@ -13,11 +23,16 @@ async def create_product(link: str) -> Product:
|
|||
product_url = get_product_details_url(product_id)
|
||||
product, data = None, await scrape_woolies_data(product_url)
|
||||
if data:
|
||||
_dump_json_data_to_log(data, product_id)
|
||||
|
||||
quantity, unit = get_package_size(data)
|
||||
product = Product(
|
||||
id=0,
|
||||
product_id=product_id,
|
||||
name=data['Product']['Name'],
|
||||
link=link,
|
||||
quantity=quantity,
|
||||
unit=unit,
|
||||
img_small=data['Product']['SmallImageFile'],
|
||||
img_large=data['Product']['LargeImageFile'],
|
||||
)
|
||||
|
|
@ -54,3 +69,17 @@ async def get_or_create(conn, url: str, tags: List[str]) -> Product:
|
|||
await add_missing_tags(conn, product, tags)
|
||||
|
||||
return product
|
||||
|
||||
def _dump_json_data_to_log(data: dict, product_id: str) -> str:
|
||||
import os, re
|
||||
dir = './data/dump'
|
||||
if not os.path.exists(dir):
|
||||
os.makedirs(dir)
|
||||
|
||||
prefix = f'product_{product_id}'
|
||||
suffix = '.json'
|
||||
file_ids = [int(re.findall(r'\d+', f)[0]) for f in os.listdir(dir) if re.match(prefix + r'\d+' + suffix, f)]
|
||||
id = max(file_ids) + 1 if file_ids else 0
|
||||
filename = f'{prefix}{id}{suffix}'
|
||||
with open(os.path.join(dir, filename), 'w') as f:
|
||||
json.dump(data, f, indent=4)
|
||||
|
|
@ -4,11 +4,15 @@ from pydantic import BaseModel
|
|||
import json
|
||||
|
||||
class Product(BaseModel):
|
||||
KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large']
|
||||
KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'quantity', 'unit', 'img_small', 'img_large']
|
||||
NON_INSERT_KEYS: ClassVar[List[str]] = ['id']
|
||||
|
||||
id: int
|
||||
product_id: str
|
||||
link: str
|
||||
name: str
|
||||
quantity: int
|
||||
unit: str
|
||||
img_small: str
|
||||
img_large: str
|
||||
|
||||
|
|
@ -19,6 +23,8 @@ async def create(conn):
|
|||
product_id TEXT UNIQUE,
|
||||
link TEXT,
|
||||
name TEXT,
|
||||
quantity INTEGER,
|
||||
unit TEXT,
|
||||
img_small TEXT,
|
||||
img_large TEXT,
|
||||
raw_data TEXT
|
||||
|
|
@ -62,10 +68,13 @@ async def find_product_by_product_id(conn, product_id: str) -> Product:
|
|||
return Product(**{k:v for k,v in zip(Product.KEYS, row)})
|
||||
|
||||
async def insert_product(conn, product: Product, data: dict):
|
||||
async with conn.execute('''
|
||||
INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
''', (product.name, product.product_id, product.link, product.img_small, product.img_large, json.dumps(data))) as cursor:
|
||||
insert_keys = [k for k in Product.KEYS if k not in Product.NON_INSERT_KEYS]
|
||||
insert_values = [getattr(product, k) for k in insert_keys]
|
||||
|
||||
async with conn.execute(f'''
|
||||
INSERT INTO Product ({','.join(insert_keys)}, raw_data)
|
||||
VALUES ({','.join(['?'] * len(insert_keys))}, ?)
|
||||
''', (*insert_values, json.dumps(data))) as cursor:
|
||||
product.id = cursor.lastrowid
|
||||
|
||||
await conn.commit()
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ class Products:
|
|||
id=0,
|
||||
name="Fresh Broccoli",
|
||||
product_id="134681",
|
||||
quantity=1,
|
||||
unit="Items",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/134681/fresh-broccoli",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/134681.jpg",
|
||||
img_large="https://cdn0.woolworths.media/content/wowproductimages/large/134681.jpg",
|
||||
|
|
@ -34,6 +36,8 @@ class Products:
|
|||
id=0,
|
||||
name="La Famiglia Garlic Bread",
|
||||
product_id="294517",
|
||||
quantity=1,
|
||||
unit="Loaf",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/294517/la-famiglia-garlic-bread",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/294517.jpg",
|
||||
img_large="https://cdn0.woolworths.media/content/wowproductimages/large/294517.jpg",
|
||||
|
|
@ -44,6 +48,8 @@ class Products:
|
|||
id=0,
|
||||
name="Beans Round",
|
||||
product_id="134072",
|
||||
quantity=1,
|
||||
unit="kg",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/134072/beans-round",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/134072.jpg",
|
||||
img_large="https://cdn0.woolworths.media/content/wowproductimages/large/134072.jpg",
|
||||
|
|
@ -54,6 +60,8 @@ class Products:
|
|||
id=0,
|
||||
name="Western Star Unsalted Butter Chef's Choice",
|
||||
product_id="712251",
|
||||
quantity=500,
|
||||
unit="g",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/712251/western-star-unsalted-butter-chef-s-choice",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/712251.jpg",
|
||||
img_large="https://cdn0.woolworths.media/content/wowproductimages/large/712251.jpg",
|
||||
|
|
@ -63,6 +71,8 @@ class Products:
|
|||
saxa_iodised_table_salt_shaker = products.Product(
|
||||
id=0,
|
||||
name="Saxa Iodised Table Salt Shaker",
|
||||
quantity=750,
|
||||
unit="g",
|
||||
product_id="33245",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/33245/saxa-iodised-table-salt-shaker",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/033245.jpg",
|
||||
|
|
@ -73,6 +83,8 @@ class Products:
|
|||
mckenzies_pepper_black_ground = products.Product(
|
||||
id=0,
|
||||
name="Mckenzie's Pepper Black Ground",
|
||||
quantity=100,
|
||||
unit="g",
|
||||
product_id="75194",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/75194/mckenzie-s-pepper-black-ground",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/075194.jpg",
|
||||
|
|
@ -84,6 +96,8 @@ class Products:
|
|||
id=0,
|
||||
name="Apple",
|
||||
product_id="3542",
|
||||
quantity=1,
|
||||
unit="Items",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/0/apple",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/0.jpg",
|
||||
img_large="https://cdn0.woolworths.media/content/wowproductimages/large/0.jpg",
|
||||
|
|
@ -94,6 +108,8 @@ class Products:
|
|||
id=0,
|
||||
name="Banana",
|
||||
product_id="214",
|
||||
quantity=1,
|
||||
unit="Items",
|
||||
link="https://www.woolworths.com.au/shop/productdetails/0/banana",
|
||||
img_small="https://cdn0.woolworths.media/content/wowproductimages/small/0.jpg",
|
||||
img_large="https://cdn0.woolworths.media/content/wowproductimages/large/0.jpg",
|
||||
|
|
|
|||
Loading…
Reference in a new issue