Significant refactor

db.py

@@ -1,56 +1,15 @@
-import sqlite3
 import aiosqlite
 
-from model import *
-from typing import List
+async def connect() -> aiosqlite.Connection:
+    return await aiosqlite.connect('your_database.db')
 
 async def create():
+    import product.db as product_db
     conn = await connect()
-    await conn.execute('''
-        CREATE TABLE IF NOT EXISTS Product (
-            id INTEGER PRIMARY KEY,
-            name TEXT,
-            link TEXT UNIQUE,
-            img_small TEXT,
-            img_large TEXT,
-            raw_data TEXT
-        );''')
-    
-    await conn.execute('''       
-        CREATE TABLE IF NOT EXISTS ProductTag (
-            food_item_id INTEGER,
-            tag TEXT,
-            PRIMARY KEY (food_item_id, tag),
-            FOREIGN KEY (food_item_id) REFERENCES Product(id)
-        );''')
-
-    # Commit changes and close the connection
+    await product_db.create(conn)
     await conn.commit()
     await conn.close()
 
-async def find_product_by_tag(conn, tag: str) -> List[Product]:
-    async with conn.execute('''
-        SELECT * FROM Product
-        WHERE id IN (
-            SELECT food_item_id FROM ProductTag
-            WHERE tag = ?
-        )
-    ''', (tag,)) as cursor:
-        async for row in cursor:
-            yield Product(*row)
-
-async def insert_product(conn, product: Product):
-    async with conn.execute('''
-        INSERT INTO Product (name, link, img_small, img_large, raw_data)
-        VALUES (?, ?, ?, ?, ?)
-    ''', (product.name, product.link, product.img_small, product.img_large, product.raw_data)) as cursor:
-        product.id = cursor.lastrowid
-    
-    await conn.commit()
-
-async def connect():
-    return await aiosqlite.connect('your_database.db')
-
 if __name__ == '__main__':
     import asyncio
     asyncio.run(create())

main.py

@@ -1,8 +1,9 @@
 import sqlite3
-import product, db, recipe
+import product, recipe, db
 
-from typing import List
-from fastapi import FastAPI, Depends, HTTPException
+from pydantic import BaseModel
+from typing import List, Annotated
+from fastapi import FastAPI, Depends, Query
 from fastapi.middleware.cors import CORSMiddleware
 
 app = FastAPI()
@@ -22,20 +23,26 @@ async def get_db():
     try:
         yield sql_db
     finally:
-        sql_db.close()
+        await sql_db.close()
 
 @app.get("/recipes/parse")
 async def parse_recipe_handler(url: str, conn: sqlite3.Connection = Depends(get_db)):
     return await recipe.parse_recipe(conn, url)
 
 @app.get("/recipes/ingredients/parse")
-async def parse_ingredients(lines: List[str], conn: sqlite3.Connection = Depends(get_db)):
+async def parse_ingredients(lines: Annotated[
+            List[str], 
+            Query(alias="ingredients",
+                title="Array of ingredients to parse")], 
+        conn: sqlite3.Connection = Depends(get_db)):
     ingredients = recipe.parse_ingredient_from_nlp(lines)
     recipe.match_existing_products(conn, ingredients)
     return ingredients
 
-@app.post("/product/")
-async def create_product(url: str, conn: sqlite3.Connection = Depends(get_db)):
-    p = await product.create_product(url)
-    db.insert_product(conn, p)
-    return p
+class ProductUrl(BaseModel):
+    url: str
+    tags: List[str] = []
+
+@app.post("/products/")
+async def create_product(url: ProductUrl, conn: sqlite3.Connection = Depends(get_db)):
+    return await product.get_or_create(conn, url.url, url.tags)

model.py

@@ -1,14 +1,7 @@
 from typing import List, Union
 from datetime import datetime
 
-class Product:
-    def __init__(self, id: int, name: str, link: str, img_small: str, img_large: str, raw_data: dict) -> None:
-        self.id = id
-        self.name = name
-        self.link = link
-        self.img_small = img_small
-        self.img_large = img_large
-        self.raw_data = raw_data
+from product import Product
 
 class Person:
     def __init__(self, id: int, name: str) -> None:
@@ -21,12 +14,12 @@ class Measurement:
         self.unit = unit
 
 class Ingredient:
-    def __init__(self, id: int, source: str, name: str, product: Product, measure: Measurement, preparation: str) -> None:
+    def __init__(self, id: int, line: str, name: str, product: Product, measure: Measurement, preparation: str) -> None:
         self.id = id
         self.name = name
         self.product = product
         self.measure = measure
-        self.source = source
+        self.line = line
         self.preparation = preparation
 
 class Recipe:

plan.md

@@ -0,0 +1,38 @@
+
+# Goals
+* Reduce effort for woolies
+* Be asynchronous - don't have to wait for people
+* Be shared - everyone can see what's for dinner
+
+# Features
+## Jacob
+* User can paste a link to add a recipe
+  * Recipe ingredients are automatically pulled
+  * User must specify which woolworths product matches each ingredient
+    * Auto-match where possible from existing saved recipes and products
+  * User may override parsed data to fix errors
+
+* I can organise our meal plan (which contains the specific products chosen)
+  * Include recipe, date, who is cooking & who is eating
+  * Allow swapping meal days (eg, swap Mon & Tue) - Ellie -> can drag and drop the plan and get confirmation of day change or use an arrow on side of card
+  * Allow changing details of meal plan
+
+* I can build a shopping list from meal plan
+  * Products automatically added based off meal plan
+  * I can see the source of each product (user-added, or specific meal)
+  * Specific product links, not just 'Beef mince'
+
+## Everyone
+* Users can browse meal plan
+  * HA integration - shown on HA overview - Home assistant
+  * Link to recipe easy click 
+
+* Users can add to next shopping list
+  * HA Integration
+  * Specify exact product and quantity
+  
+  # Available
+
+  ## Services
+  Recipe -> Ingredients (with product matching of existing products)
+  

product.py

@@ -1,26 +0,0 @@
-import httpx
-import re
-from model import Product
-import json
-from typing import List
-
-async def _get_woolies_data(url: str) -> dict:
-    woolies_regex = r'https://www.woolworths.com.au/shop/productdetails/(\d+)/'
-    match = re.match(woolies_regex, url)
-    if match:
-        product_id = match.group(1)
-        async with httpx.AsyncClient() as client:
-            response = await client.get(f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}')
-            response.raise_for_status()
-            return response.json()
-
-async def create_product(url: str) -> Product:
-    data = await _get_woolies_data(url)
-    return Product(
-        id=0,
-        name=data['Name'],
-        link=url,
-        img_small=data['SmallImageFile'],
-        img_large=data['LargeImageFile'],
-        raw_data=json.dumps(data)
-    )

product/__init__.py

@@ -0,0 +1,55 @@
+import json
+
+from product.db import Product, find_product_by_tag, find_product_by_product_id, insert_product, get_tags, add_tag
+from product.scraping import scrape_woolies_data, get_product_id, get_product_details_url
+
+from typing import List
+
+async def create_product(link: str) -> Product:
+    product_id = get_product_id(link)
+    if not product_id:
+        return None
+    
+    product_url = get_product_details_url(product_id)
+    data = await scrape_woolies_data(product_url)
+    if data:
+        return Product(
+            id=0,
+            product_id=product_id,
+            name=data['Product']['Name'],
+            link=link,
+            img_small=data['Product']['SmallImageFile'],
+            img_large=data['Product']['LargeImageFile'],
+            raw_data=json.dumps(data)
+        )
+    
+async def add_missing_tags(conn, product: Product, tags: List[str]):
+    existing_tags = set()
+    async for tag in get_tags(conn, product):
+        existing_tags.add(tag)
+    
+    remaining_tags = set(tags) - existing_tags
+    if not remaining_tags:
+        return False
+    
+    for tag in remaining_tags:
+        await add_tag(conn, product, tag)
+    
+    return product
+
+async def get_or_create(conn, url: str, tags: List[str]) -> Product:
+    product_id = get_product_id(url)
+    if not product_id:
+        return None
+    
+    existing = await find_product_by_product_id(conn, product_id)
+    if existing:
+        await add_missing_tags(conn, existing, tags)
+        return existing
+    
+    product = await create_product(url)
+    if product:
+        await insert_product(conn, product)
+        await add_missing_tags(conn, product, tags)
+
+    return product

product/db.py

@@ -0,0 +1,81 @@
+import aiosqlite
+
+from typing import List
+
+class Product:
+    KEYS = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large', 'raw_data']
+
+    def __init__(self, id: int, product_id: str, link: str, name: str, img_small: str, img_large: str, raw_data: dict) -> None:
+        self.id = id
+        self.product_id = product_id
+        self.link = link
+        self.name = name
+        self.img_small = img_small
+        self.img_large = img_large
+        self.raw_data = raw_data
+
+async def create(conn):
+    await conn.execute('''
+        CREATE TABLE IF NOT EXISTS Product (
+            id INTEGER PRIMARY KEY,
+            product_id TEXT UNIQUE,
+            link TEXT,
+            name TEXT,
+            img_small TEXT,
+            img_large TEXT,
+            raw_data TEXT
+        );''')
+    
+    await conn.execute('''       
+        CREATE TABLE IF NOT EXISTS ProductTag (
+            food_item_id INTEGER,
+            tag TEXT,
+            PRIMARY KEY (food_item_id, tag),
+            FOREIGN KEY (food_item_id) REFERENCES Product(id)
+        );''')
+
+async def find_product_by_tag(conn, tag: str) -> List[Product]:
+    async with conn.execute(f'''
+        SELECT {','.join(Product.KEYS)} FROM Product
+        WHERE id IN (
+            SELECT food_item_id FROM ProductTag
+            WHERE tag = ?
+        )
+    ''', (tag,)) as cursor:
+        async for row in cursor:
+            yield Product(*row)
+
+async def find_product_by_product_id(conn, product_id: str) -> Product:
+    async with conn.execute(f'''
+        SELECT {','.join(Product.KEYS)} FROM Product
+        WHERE product_id = ?
+        LIMIT 1
+    ''', (product_id,)) as cursor:
+        async for row in cursor:
+            return Product(*row)
+
+async def insert_product(conn, product: Product):
+    async with conn.execute('''
+        INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data)
+        VALUES (?, ?, ?, ?, ?, ?)
+    ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, product.raw_data)) as cursor:
+        product.id = cursor.lastrowid
+    
+    await conn.commit()
+
+async def add_tag(conn, product: Product, tag: str):
+    await conn.execute('''
+        INSERT INTO ProductTag (food_item_id, tag)
+        VALUES (?, ?)
+    ''', (product.id, tag))
+    
+    await conn.commit()
+
+async def get_tags(conn, product: Product) -> List[str]:
+    async with conn.execute('''
+        SELECT tag FROM ProductTag
+        WHERE food_item_id = ?
+    ''', (product.id,)) as cursor:
+        async for row in cursor:
+            yield row[0]
+

product/scraping.py

@@ -0,0 +1,54 @@
+import re, httpx
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'DNT': '1',
+    'Sec-GPC': '1',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'Pragma': 'no-cache',
+    'Cache-Control': 'no-cache',
+}
+
+async def _get_cookies(client):
+    # Make a request to https://www.woolworths.com.au/ as if we were a normal browser, then return the cookies
+    response = await client.get('https://www.woolworths.com.au/', headers=HEADERS, follow_redirects=True)
+    return dict(response.cookies)
+
+last_cookies = None
+async def scrape_woolies_data(url: str) -> dict:
+    global last_cookies
+
+    async with httpx.AsyncClient() as client:
+        if last_cookies:
+            cookies = last_cookies
+        else:
+            cookies = await _get_cookies(client)
+            last_cookies = cookies
+
+        try:
+            response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies)
+            response.raise_for_status()
+            return response.json()
+        except httpx.NetworkError as ne:
+            print(ne)
+            last_cookies = None
+            return None
+
+def get_product_id(url: str) -> str:
+    woolies_regex = r'https://www.woolworths.com.au/shop/productdetails/(\d+)/?.*'
+    match = re.match(woolies_regex, url)
+    if match:
+        return match.group(1)
+    return None
+
+def get_product_details_url(product_id) -> str:
+    return f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}'
+

recipe.py

@@ -5,7 +5,7 @@ from bs4 import BeautifulSoup
 
 import httpx
 import json
-import db
+import product.db as db
 
 async def find_existing_product(conn, ingredient: str) -> Product:
     async for item in db.find_product_by_tag(conn, ingredient):
@@ -15,7 +15,8 @@ async def find_existing_product(conn, ingredient: str) -> Product:
 def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient:
     results = []
     for ingredient in parse_multiple_ingredients(ingredients):
-        name = ingredient.name.text
+        name = ingredient.name.text if ingredient.name else ''
+
         if ingredient.amount:
             measure = Measurement(ingredient.amount[0].quantity, ingredient.amount[0].unit)
         else:
@@ -23,7 +24,7 @@ def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient:
 
         results.append(Ingredient(
             id=0,
-            source=ingredient.sentence,
+            line=ingredient.sentence,
             name=name,
             product=None,
             measure=measure,
@@ -59,7 +60,7 @@ async def parse_recipe(conn, url: str) -> dict:
 
     # Load the requested URL with headers
     async with httpx.AsyncClient() as client:
-        response = await client.get(url, headers=headers)
+        response = await client.get(url, headers=headers, follow_redirects=True)
         response.raise_for_status()
 
         # Extract the recipe ld+json data
@@ -67,9 +68,30 @@ async def parse_recipe(conn, url: str) -> dict:
         for ld in soup.find_all('script', type='application/ld+json'):
             try:
                 data = json.loads(ld.text)
-                if data['@type'].lower() == 'recipe':
+                #_dump_json_data_to_log(data)
+                if '@type' in data and data['@type'].lower() == 'recipe':
                     return { 'raw': data, 'recipe': await get_recipe_from_ldata(conn, url, data) }
+                
+                if '@graph' in data:
+                    for item in data['@graph']:
+                        if '@type' in item and item['@type'].lower() == 'recipe':
+                            return { 'raw': data, 'recipe': await get_recipe_from_ldata(conn, url, item) }
+
             except (json.decoder.JSONDecodeError, KeyError):
                 pass
         
-        return None
+        return None
+    
+def _dump_json_data_to_log(data: dict) -> str:
+    import os, re
+    dir = './dump'
+    if not os.path.exists(dir):
+        os.makedirs(dir)
+
+    prefix = 'ldata_'
+    suffix = '.json'
+    file_ids = [int(re.findall(r'\d+', f)[0]) for f in os.listdir(dir) if re.match(prefix + r'\d+' + suffix, f)]
+    id = max(file_ids) + 1 if file_ids else 0
+    filename = f'{prefix}{id}{suffix}'
+    with open(os.path.join(dir, filename), 'w') as f:
+        json.dump(data, f, indent=4)