From 0befcbe01e48374c01156cad93bee02a91fff082 Mon Sep 17 00:00:00 2001 From: jableader Date: Sat, 13 Jan 2024 12:54:04 +1100 Subject: [PATCH] Significant refactor --- db.py | 49 +++------------------------ main.py | 27 +++++++++------ model.py | 13 ++------ plan.md | 38 +++++++++++++++++++++ product.py | 26 --------------- product/__init__.py | 55 ++++++++++++++++++++++++++++++ product/db.py | 81 +++++++++++++++++++++++++++++++++++++++++++++ product/scraping.py | 54 ++++++++++++++++++++++++++++++ recipe.py | 34 +++++++++++++++---- 9 files changed, 280 insertions(+), 97 deletions(-) create mode 100644 plan.md delete mode 100644 product.py create mode 100644 product/__init__.py create mode 100644 product/db.py create mode 100644 product/scraping.py diff --git a/db.py b/db.py index ca93e2a..4eb10c2 100644 --- a/db.py +++ b/db.py @@ -1,56 +1,15 @@ -import sqlite3 import aiosqlite -from model import * -from typing import List +async def connect() -> aiosqlite.Connection: + return await aiosqlite.connect('your_database.db') async def create(): + import product.db as product_db conn = await connect() - await conn.execute(''' - CREATE TABLE IF NOT EXISTS Product ( - id INTEGER PRIMARY KEY, - name TEXT, - link TEXT UNIQUE, - img_small TEXT, - img_large TEXT, - raw_data TEXT - );''') - - await conn.execute(''' - CREATE TABLE IF NOT EXISTS ProductTag ( - food_item_id INTEGER, - tag TEXT, - PRIMARY KEY (food_item_id, tag), - FOREIGN KEY (food_item_id) REFERENCES Product(id) - );''') - - # Commit changes and close the connection + await product_db.create(conn) await conn.commit() await conn.close() -async def find_product_by_tag(conn, tag: str) -> List[Product]: - async with conn.execute(''' - SELECT * FROM Product - WHERE id IN ( - SELECT food_item_id FROM ProductTag - WHERE tag = ? - ) - ''', (tag,)) as cursor: - async for row in cursor: - yield Product(*row) - -async def insert_product(conn, product: Product): - async with conn.execute(''' - INSERT INTO Product (name, link, img_small, img_large, raw_data) - VALUES (?, ?, ?, ?, ?) - ''', (product.name, product.link, product.img_small, product.img_large, product.raw_data)) as cursor: - product.id = cursor.lastrowid - - await conn.commit() - -async def connect(): - return await aiosqlite.connect('your_database.db') - if __name__ == '__main__': import asyncio asyncio.run(create()) \ No newline at end of file diff --git a/main.py b/main.py index 5db419d..86bda7c 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,9 @@ import sqlite3 -import product, db, recipe +import product, recipe, db -from typing import List -from fastapi import FastAPI, Depends, HTTPException +from pydantic import BaseModel +from typing import List, Annotated +from fastapi import FastAPI, Depends, Query from fastapi.middleware.cors import CORSMiddleware app = FastAPI() @@ -22,20 +23,26 @@ async def get_db(): try: yield sql_db finally: - sql_db.close() + await sql_db.close() @app.get("/recipes/parse") async def parse_recipe_handler(url: str, conn: sqlite3.Connection = Depends(get_db)): return await recipe.parse_recipe(conn, url) @app.get("/recipes/ingredients/parse") -async def parse_ingredients(lines: List[str], conn: sqlite3.Connection = Depends(get_db)): +async def parse_ingredients(lines: Annotated[ + List[str], + Query(alias="ingredients", + title="Array of ingredients to parse")], + conn: sqlite3.Connection = Depends(get_db)): ingredients = recipe.parse_ingredient_from_nlp(lines) recipe.match_existing_products(conn, ingredients) return ingredients -@app.post("/product/") -async def create_product(url: str, conn: sqlite3.Connection = Depends(get_db)): - p = await product.create_product(url) - db.insert_product(conn, p) - return p +class ProductUrl(BaseModel): + url: str + tags: List[str] = [] + +@app.post("/products/") +async def create_product(url: ProductUrl, conn: sqlite3.Connection = Depends(get_db)): + return await product.get_or_create(conn, url.url, url.tags) \ No newline at end of file diff --git a/model.py b/model.py index 0c8b470..b3ee126 100644 --- a/model.py +++ b/model.py @@ -1,14 +1,7 @@ from typing import List, Union from datetime import datetime -class Product: - def __init__(self, id: int, name: str, link: str, img_small: str, img_large: str, raw_data: dict) -> None: - self.id = id - self.name = name - self.link = link - self.img_small = img_small - self.img_large = img_large - self.raw_data = raw_data +from product import Product class Person: def __init__(self, id: int, name: str) -> None: @@ -21,12 +14,12 @@ class Measurement: self.unit = unit class Ingredient: - def __init__(self, id: int, source: str, name: str, product: Product, measure: Measurement, preparation: str) -> None: + def __init__(self, id: int, line: str, name: str, product: Product, measure: Measurement, preparation: str) -> None: self.id = id self.name = name self.product = product self.measure = measure - self.source = source + self.line = line self.preparation = preparation class Recipe: diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..dec2a2d --- /dev/null +++ b/plan.md @@ -0,0 +1,38 @@ + +# Goals +* Reduce effort for woolies +* Be asynchronous - don't have to wait for people +* Be shared - everyone can see what's for dinner + +# Features +## Jacob +* User can paste a link to add a recipe + * Recipe ingredients are automatically pulled + * User must specify which woolworths product matches each ingredient + * Auto-match where possible from existing saved recipes and products + * User may override parsed data to fix errors + +* I can organise our meal plan (which contains the specific products chosen) + * Include recipe, date, who is cooking & who is eating + * Allow swapping meal days (eg, swap Mon & Tue) - Ellie -> can drag and drop the plan and get confirmation of day change or use an arrow on side of card + * Allow changing details of meal plan + +* I can build a shopping list from meal plan + * Products automatically added based off meal plan + * I can see the source of each product (user-added, or specific meal) + * Specific product links, not just 'Beef mince' + +## Everyone +* Users can browse meal plan + * HA integration - shown on HA overview - Home assistant + * Link to recipe easy click + +* Users can add to next shopping list + * HA Integration + * Specify exact product and quantity + + # Available + + ## Services + Recipe -> Ingredients (with product matching of existing products) + \ No newline at end of file diff --git a/product.py b/product.py deleted file mode 100644 index ff3e578..0000000 --- a/product.py +++ /dev/null @@ -1,26 +0,0 @@ -import httpx -import re -from model import Product -import json -from typing import List - -async def _get_woolies_data(url: str) -> dict: - woolies_regex = r'https://www.woolworths.com.au/shop/productdetails/(\d+)/' - match = re.match(woolies_regex, url) - if match: - product_id = match.group(1) - async with httpx.AsyncClient() as client: - response = await client.get(f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}') - response.raise_for_status() - return response.json() - -async def create_product(url: str) -> Product: - data = await _get_woolies_data(url) - return Product( - id=0, - name=data['Name'], - link=url, - img_small=data['SmallImageFile'], - img_large=data['LargeImageFile'], - raw_data=json.dumps(data) - ) \ No newline at end of file diff --git a/product/__init__.py b/product/__init__.py new file mode 100644 index 0000000..1479f18 --- /dev/null +++ b/product/__init__.py @@ -0,0 +1,55 @@ +import json + +from product.db import Product, find_product_by_tag, find_product_by_product_id, insert_product, get_tags, add_tag +from product.scraping import scrape_woolies_data, get_product_id, get_product_details_url + +from typing import List + +async def create_product(link: str) -> Product: + product_id = get_product_id(link) + if not product_id: + return None + + product_url = get_product_details_url(product_id) + data = await scrape_woolies_data(product_url) + if data: + return Product( + id=0, + product_id=product_id, + name=data['Product']['Name'], + link=link, + img_small=data['Product']['SmallImageFile'], + img_large=data['Product']['LargeImageFile'], + raw_data=json.dumps(data) + ) + +async def add_missing_tags(conn, product: Product, tags: List[str]): + existing_tags = set() + async for tag in get_tags(conn, product): + existing_tags.add(tag) + + remaining_tags = set(tags) - existing_tags + if not remaining_tags: + return False + + for tag in remaining_tags: + await add_tag(conn, product, tag) + + return product + +async def get_or_create(conn, url: str, tags: List[str]) -> Product: + product_id = get_product_id(url) + if not product_id: + return None + + existing = await find_product_by_product_id(conn, product_id) + if existing: + await add_missing_tags(conn, existing, tags) + return existing + + product = await create_product(url) + if product: + await insert_product(conn, product) + await add_missing_tags(conn, product, tags) + + return product diff --git a/product/db.py b/product/db.py new file mode 100644 index 0000000..e989a21 --- /dev/null +++ b/product/db.py @@ -0,0 +1,81 @@ +import aiosqlite + +from typing import List + +class Product: + KEYS = ['id', 'product_id', 'link', 'name', 'img_small', 'img_large', 'raw_data'] + + def __init__(self, id: int, product_id: str, link: str, name: str, img_small: str, img_large: str, raw_data: dict) -> None: + self.id = id + self.product_id = product_id + self.link = link + self.name = name + self.img_small = img_small + self.img_large = img_large + self.raw_data = raw_data + +async def create(conn): + await conn.execute(''' + CREATE TABLE IF NOT EXISTS Product ( + id INTEGER PRIMARY KEY, + product_id TEXT UNIQUE, + link TEXT, + name TEXT, + img_small TEXT, + img_large TEXT, + raw_data TEXT + );''') + + await conn.execute(''' + CREATE TABLE IF NOT EXISTS ProductTag ( + food_item_id INTEGER, + tag TEXT, + PRIMARY KEY (food_item_id, tag), + FOREIGN KEY (food_item_id) REFERENCES Product(id) + );''') + +async def find_product_by_tag(conn, tag: str) -> List[Product]: + async with conn.execute(f''' + SELECT {','.join(Product.KEYS)} FROM Product + WHERE id IN ( + SELECT food_item_id FROM ProductTag + WHERE tag = ? + ) + ''', (tag,)) as cursor: + async for row in cursor: + yield Product(*row) + +async def find_product_by_product_id(conn, product_id: str) -> Product: + async with conn.execute(f''' + SELECT {','.join(Product.KEYS)} FROM Product + WHERE product_id = ? + LIMIT 1 + ''', (product_id,)) as cursor: + async for row in cursor: + return Product(*row) + +async def insert_product(conn, product: Product): + async with conn.execute(''' + INSERT INTO Product (name, product_id, link, img_small, img_large, raw_data) + VALUES (?, ?, ?, ?, ?, ?) + ''', (product.name, product.product_id, product.link, product.img_small, product.img_large, product.raw_data)) as cursor: + product.id = cursor.lastrowid + + await conn.commit() + +async def add_tag(conn, product: Product, tag: str): + await conn.execute(''' + INSERT INTO ProductTag (food_item_id, tag) + VALUES (?, ?) + ''', (product.id, tag)) + + await conn.commit() + +async def get_tags(conn, product: Product) -> List[str]: + async with conn.execute(''' + SELECT tag FROM ProductTag + WHERE food_item_id = ? + ''', (product.id,)) as cursor: + async for row in cursor: + yield row[0] + diff --git a/product/scraping.py b/product/scraping.py new file mode 100644 index 0000000..0c50473 --- /dev/null +++ b/product/scraping.py @@ -0,0 +1,54 @@ +import re, httpx + +HEADERS = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate, br', + 'DNT': '1', + 'Sec-GPC': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'Pragma': 'no-cache', + 'Cache-Control': 'no-cache', +} + +async def _get_cookies(client): + # Make a request to https://www.woolworths.com.au/ as if we were a normal browser, then return the cookies + response = await client.get('https://www.woolworths.com.au/', headers=HEADERS, follow_redirects=True) + return dict(response.cookies) + +last_cookies = None +async def scrape_woolies_data(url: str) -> dict: + global last_cookies + + async with httpx.AsyncClient() as client: + if last_cookies: + cookies = last_cookies + else: + cookies = await _get_cookies(client) + last_cookies = cookies + + try: + response = await client.get(url, headers=HEADERS, follow_redirects=True, cookies=cookies) + response.raise_for_status() + return response.json() + except httpx.NetworkError as ne: + print(ne) + last_cookies = None + return None + +def get_product_id(url: str) -> str: + woolies_regex = r'https://www.woolworths.com.au/shop/productdetails/(\d+)/?.*' + match = re.match(woolies_regex, url) + if match: + return match.group(1) + return None + +def get_product_details_url(product_id) -> str: + return f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}' + diff --git a/recipe.py b/recipe.py index 9d84fa8..def90b8 100644 --- a/recipe.py +++ b/recipe.py @@ -5,7 +5,7 @@ from bs4 import BeautifulSoup import httpx import json -import db +import product.db as db async def find_existing_product(conn, ingredient: str) -> Product: async for item in db.find_product_by_tag(conn, ingredient): @@ -15,7 +15,8 @@ async def find_existing_product(conn, ingredient: str) -> Product: def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient: results = [] for ingredient in parse_multiple_ingredients(ingredients): - name = ingredient.name.text + name = ingredient.name.text if ingredient.name else '' + if ingredient.amount: measure = Measurement(ingredient.amount[0].quantity, ingredient.amount[0].unit) else: @@ -23,7 +24,7 @@ def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient: results.append(Ingredient( id=0, - source=ingredient.sentence, + line=ingredient.sentence, name=name, product=None, measure=measure, @@ -59,7 +60,7 @@ async def parse_recipe(conn, url: str) -> dict: # Load the requested URL with headers async with httpx.AsyncClient() as client: - response = await client.get(url, headers=headers) + response = await client.get(url, headers=headers, follow_redirects=True) response.raise_for_status() # Extract the recipe ld+json data @@ -67,9 +68,30 @@ async def parse_recipe(conn, url: str) -> dict: for ld in soup.find_all('script', type='application/ld+json'): try: data = json.loads(ld.text) - if data['@type'].lower() == 'recipe': + #_dump_json_data_to_log(data) + if '@type' in data and data['@type'].lower() == 'recipe': return { 'raw': data, 'recipe': await get_recipe_from_ldata(conn, url, data) } + + if '@graph' in data: + for item in data['@graph']: + if '@type' in item and item['@type'].lower() == 'recipe': + return { 'raw': data, 'recipe': await get_recipe_from_ldata(conn, url, item) } + except (json.decoder.JSONDecodeError, KeyError): pass - return None \ No newline at end of file + return None + +def _dump_json_data_to_log(data: dict) -> str: + import os, re + dir = './dump' + if not os.path.exists(dir): + os.makedirs(dir) + + prefix = 'ldata_' + suffix = '.json' + file_ids = [int(re.findall(r'\d+', f)[0]) for f in os.listdir(dir) if re.match(prefix + r'\d+' + suffix, f)] + id = max(file_ids) + 1 if file_ids else 0 + filename = f'{prefix}{id}{suffix}' + with open(os.path.join(dir, filename), 'w') as f: + json.dump(data, f, indent=4) \ No newline at end of file