jacob/munch-ease-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove product raw data from responses

Browse source
This commit is contained in:
jableader 2024-01-18 19:28:26 +11:00
parent 24f97230f0
commit b794b6d78a
2 changed files with 12 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
products/__init__.py
View file

@ -8,20 +8,21 @@ from typing import List
async def create_product(link: str) -> Product: async def create_product(link: str) -> Product:
product_id = get_product_id(link) product_id = get_product_id(link)
if not product_id: if not product_id:
return None return None, None
product_url = get_product_details_url(product_id) product_url = get_product_details_url(product_id)
data = await scrape_woolies_data(product_url) product, data = None, await scrape_woolies_data(product_url)
if data: if data:
return Product( product = Product(
id=0, id=0,
product_id=product_id, product_id=product_id,
name=data['Product']['Name'], name=data['Product']['Name'],
link=link, link=link,
img_small=data['Product']['SmallImageFile'], img_small=data['Product']['SmallImageFile'],
img_large=data['Product']['LargeImageFile'], img_large=data['Product']['LargeImageFile'],
raw_data=json.dumps(data)
) )
return product, data
async def add_missing_tags(conn, product: Product, tags: List[str]): async def add_missing_tags(conn, product: Product, tags: List[str]):
existing_tags = set() existing_tags = set()
@ -47,9 +48,9 @@ async def get_or_create(conn, url: str, tags: List[str]) -> Product:
await add_missing_tags(conn, existing, tags) await add_missing_tags(conn, existing, tags)
return existing return existing
product = await create_product(url) product, data = await create_product(url)
if product: if product:
await insert_product(conn, product) await insert_product(conn, product, data)
await add_missing_tags(conn, product, tags) await add_missing_tags(conn, product, tags)
return product return product

9
products/db.py
View file

@ -1,15 +1,16 @@
from typing import List, ClassVar from typing import List, ClassVar
from pydantic import BaseModel from pydantic import BaseModel
import json
class Product(BaseModel): class Product(BaseModel):
KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large', 'raw_data'] KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large']
id: int id: int
product_id: str product_id: str
link: str link: str
name: str name: str
img_small: str img_small: str
img_large: str img_large: str
raw_data: str
async def create(conn): async def create(conn):
await conn.execute(''' await conn.execute('''
@ -60,11 +61,11 @@ async def find_product_by_product_id(conn, product_id: str) -> Product:
async for row in cursor: async for row in cursor:
return Product(**{k:v for k,v in zip(Product.KEYS, row)}) return Product(**{k:v for k,v in zip(Product.KEYS, row)})
async def insert_product(conn, product: Product): async def insert_product(conn, product: Product, data: dict):
async with conn.execute(''' async with conn.execute('''
INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data) INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data)
VALUES (?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?)
''', (product.name, product.product_id, product.link, product.img_small, product.img_large, product.raw_data)) as cursor: ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, json.dumps(data))) as cursor:
product.id = cursor.lastrowid product.id = cursor.lastrowid
await conn.commit() await conn.commit()