Remove product raw data from responses

products/__init__.py

@@ -8,20 +8,21 @@ from typing import List
 async def create_product(link: str) -> Product:
     product_id = get_product_id(link)
     if not product_id:
-        return None
+        return None, None
     
     product_url = get_product_details_url(product_id)
-    data = await scrape_woolies_data(product_url)
+    product, data = None, await scrape_woolies_data(product_url)
     if data:
-        return Product(
+        product =  Product(
             id=0,
             product_id=product_id,
             name=data['Product']['Name'],
             link=link,
             img_small=data['Product']['SmallImageFile'],
             img_large=data['Product']['LargeImageFile'],
-            raw_data=json.dumps(data)
         )
+
+    return product, data
     
 async def add_missing_tags(conn, product: Product, tags: List[str]):
     existing_tags = set()
@@ -47,9 +48,9 @@ async def get_or_create(conn, url: str, tags: List[str]) -> Product:
         await add_missing_tags(conn, existing, tags)
         return existing
     
-    product = await create_product(url)
+    product, data = await create_product(url)
     if product:
-        await insert_product(conn, product)
+        await insert_product(conn, product, data)
         await add_missing_tags(conn, product, tags)
 
     return product

products/db.py

@@ -1,15 +1,16 @@
 from typing import List, ClassVar
 from pydantic import BaseModel
 
+import json
+
 class Product(BaseModel):
-    KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large', 'raw_data']
+    KEYS: ClassVar[List[str]] = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large']
     id: int
     product_id: str
     link: str
     name: str
     img_small: str
     img_large: str
-    raw_data: str
 
 async def create(conn):
     await conn.execute('''
@@ -60,11 +61,11 @@ async def find_product_by_product_id(conn, product_id: str) -> Product:
         async for row in cursor:
             return Product(**{k:v for k,v in zip(Product.KEYS, row)})
 
-async def insert_product(conn, product: Product):
+async def insert_product(conn, product: Product, data: dict):
     async with conn.execute('''
         INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data)
         VALUES (?, ?, ?, ?, ?, ?)
-    ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, product.raw_data)) as cursor:
+    ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, json.dumps(data))) as cursor:
         product.id = cursor.lastrowid
     
     await conn.commit()