diff --git a/products/__init__.py b/products/__init__.py index db1b73b..e36e03c 100644 --- a/products/__init__.py +++ b/products/__init__.py @@ -8,20 +8,21 @@ from typing import List async def create_product(link: str) -> Product: product_id = get_product_id(link) if not product_id: - return None + return None, None product_url = get_product_details_url(product_id) - data = await scrape_woolies_data(product_url) + product, data = None, await scrape_woolies_data(product_url) if data: - return Product( + product = Product( id=0, product_id=product_id, name=data['Product']['Name'], link=link, img_small=data['Product']['SmallImageFile'], img_large=data['Product']['LargeImageFile'], - raw_data=json.dumps(data) ) + + return product, data async def add_missing_tags(conn, product: Product, tags: List[str]): existing_tags = set() @@ -47,9 +48,9 @@ async def get_or_create(conn, url: str, tags: List[str]) -> Product: await add_missing_tags(conn, existing, tags) return existing - product = await create_product(url) + product, data = await create_product(url) if product: - await insert_product(conn, product) + await insert_product(conn, product, data) await add_missing_tags(conn, product, tags) return product diff --git a/products/db.py b/products/db.py index ebd32b3..e2d4fad 100644 --- a/products/db.py +++ b/products/db.py @@ -1,15 +1,16 @@ from typing import List, ClassVar from pydantic import BaseModel +import json + class Product(BaseModel): - KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large', 'raw_data'] + KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large'] id: int product_id: str link: str name: str img_small: str img_large: str - raw_data: str async def create(conn): await conn.execute(''' @@ -60,11 +61,11 @@ async def find_product_by_product_id(conn, product_id: str) -> Product: async for row in cursor: return Product(**{k:v for k,v in zip(Product.KEYS, row)}) -async def insert_product(conn, product: Product): +async def insert_product(conn, product: Product, data: dict): async with conn.execute(''' INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data) VALUES (?, ?, ?, ?, ?, ?) - ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, product.raw_data)) as cursor: + ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, json.dumps(data))) as cursor: product.id = cursor.lastrowid await conn.commit()