diff --git a/db.py b/db.py new file mode 100644 index 0000000..ca93e2a --- /dev/null +++ b/db.py @@ -0,0 +1,56 @@ +import sqlite3 +import aiosqlite + +from model import * +from typing import List + +async def create(): + conn = await connect() + await conn.execute(''' + CREATE TABLE IF NOT EXISTS Product ( + id INTEGER PRIMARY KEY, + name TEXT, + link TEXT UNIQUE, + img_small TEXT, + img_large TEXT, + raw_data TEXT + );''') + + await conn.execute(''' + CREATE TABLE IF NOT EXISTS ProductTag ( + food_item_id INTEGER, + tag TEXT, + PRIMARY KEY (food_item_id, tag), + FOREIGN KEY (food_item_id) REFERENCES Product(id) + );''') + + # Commit changes and close the connection + await conn.commit() + await conn.close() + +async def find_product_by_tag(conn, tag: str) -> List[Product]: + async with conn.execute(''' + SELECT * FROM Product + WHERE id IN ( + SELECT food_item_id FROM ProductTag + WHERE tag = ? + ) + ''', (tag,)) as cursor: + async for row in cursor: + yield Product(*row) + +async def insert_product(conn, product: Product): + async with conn.execute(''' + INSERT INTO Product (name, link, img_small, img_large, raw_data) + VALUES (?, ?, ?, ?, ?) + ''', (product.name, product.link, product.img_small, product.img_large, product.raw_data)) as cursor: + product.id = cursor.lastrowid + + await conn.commit() + +async def connect(): + return await aiosqlite.connect('your_database.db') + +if __name__ == '__main__': + import asyncio + asyncio.run(create()) \ No newline at end of file diff --git a/main.py b/main.py index d786432..a77f074 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,34 @@ -from fastapi import FastAPI +from fastapi import FastAPI, Depends, HTTPException +from fastapi.middleware.cors import CORSMiddleware +import sqlite3 +from recipe import parse_recipe +import product, db app = FastAPI() +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"] +) -@app.get("/") -async def root(): - return {"message": "Hello World"} \ No newline at end of file +# Dependency to create SQLite connection +async def get_db(): + sql_db = await db.connect() + try: + yield sql_db + finally: + sql_db.close() + +@app.get("/recipes/parse") +async def parse_recipe_handler(url: str, conn: sqlite3.Connection = Depends(get_db)): + return await parse_recipe(conn, url) + +@app.post("/product/") +async def create_product(url: str, conn: sqlite3.Connection = Depends(get_db)): + p = await product.create_product(url) + db.insert_product(conn, p) + return p diff --git a/model.py b/model.py new file mode 100644 index 0000000..0c8b470 --- /dev/null +++ b/model.py @@ -0,0 +1,59 @@ +from typing import List, Union +from datetime import datetime + +class Product: + def __init__(self, id: int, name: str, link: str, img_small: str, img_large: str, raw_data: dict) -> None: + self.id = id + self.name = name + self.link = link + self.img_small = img_small + self.img_large = img_large + self.raw_data = raw_data + +class Person: + def __init__(self, id: int, name: str) -> None: + self.id = id + self.name = name + +class Measurement: + def __init__(self, qty: int, unit: str) -> None: + self.qty = qty + self.unit = unit + +class Ingredient: + def __init__(self, id: int, source: str, name: str, product: Product, measure: Measurement, preparation: str) -> None: + self.id = id + self.name = name + self.product = product + self.measure = measure + self.source = source + self.preparation = preparation + +class Recipe: + def __init__(self, id: int, name: str, link: str, ingredients: List[Ingredient]) -> None: + self.id = id + self.name = name + self.link = link + self.ingredients = ingredients + +class Meal: + def __init__(self, id: int, date: datetime, chef: List[Person], cleanup: List[Person], + consumers: List[Person], recipes: List[Recipe]) -> None: + self.id = id + self.date = date + self.chef = chef + self.cleanup = cleanup + self.consumers = consumers + self.recipes = recipes + +class ShoppingListItem: + def __init__(self, id: int, food_item: Product, measure: Measurement, source: Union[Meal, None]) -> None: + self.id = id + self.food_item = food_item + self.measure = measure + self.source = source + +class ShoppingList: + def __init__(self, date: datetime, items: List[ShoppingListItem]) -> None: + self.date = date + self.items = items \ No newline at end of file diff --git a/product.py b/product.py new file mode 100644 index 0000000..ff3e578 --- /dev/null +++ b/product.py @@ -0,0 +1,26 @@ +import httpx +import re +from model import Product +import json +from typing import List + +async def _get_woolies_data(url: str) -> dict: + woolies_regex = r'https://www.woolworths.com.au/shop/productdetails/(\d+)/' + match = re.match(woolies_regex, url) + if match: + product_id = match.group(1) + async with httpx.AsyncClient() as client: + response = await client.get(f'https://www.woolworths.com.au/apis/ui/product/detail/{product_id}') + response.raise_for_status() + return response.json() + +async def create_product(url: str) -> Product: + data = await _get_woolies_data(url) + return Product( + id=0, + name=data['Name'], + link=url, + img_small=data['SmallImageFile'], + img_large=data['LargeImageFile'], + raw_data=json.dumps(data) + ) \ No newline at end of file diff --git a/recipe.py b/recipe.py new file mode 100644 index 0000000..9d84fa8 --- /dev/null +++ b/recipe.py @@ -0,0 +1,75 @@ +from model import Product, Ingredient, Recipe, Measurement +from ingredient_parser import parse_multiple_ingredients +from typing import List +from bs4 import BeautifulSoup + +import httpx +import json +import db + +async def find_existing_product(conn, ingredient: str) -> Product: + async for item in db.find_product_by_tag(conn, ingredient): + return item + return None + +def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient: + results = [] + for ingredient in parse_multiple_ingredients(ingredients): + name = ingredient.name.text + if ingredient.amount: + measure = Measurement(ingredient.amount[0].quantity, ingredient.amount[0].unit) + else: + measure = Measurement(0, '') + + results.append(Ingredient( + id=0, + source=ingredient.sentence, + name=name, + product=None, + measure=measure, + preparation=ingredient.preparation.text if ingredient.preparation else '' + )) + + return results + +async def match_existing_products(conn, ingredients: List[Ingredient]) -> List[Ingredient]: + for ingredient in ingredients: + if not ingredient.product: + existing = await find_existing_product(conn, ingredient.name) + if existing: + ingredient.product = existing + return ingredients + +async def get_recipe_from_ldata(conn, url: str, ldata: dict) -> dict: + ingredients = parse_ingredient_from_nlp(ldata['recipeIngredient']) + ingredients = await match_existing_products(conn, ingredients) + return Recipe( + id=0, + name=ldata['name'], + link=url, + ingredients=ingredients + ) + +async def parse_recipe(conn, url: str) -> dict: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Accept-Language": "en-US,en;q=0.9", + "Referer": "https://www.google.com/", + } + + # Load the requested URL with headers + async with httpx.AsyncClient() as client: + response = await client.get(url, headers=headers) + response.raise_for_status() + + # Extract the recipe ld+json data + soup = BeautifulSoup(response.text, 'html.parser') + for ld in soup.find_all('script', type='application/ld+json'): + try: + data = json.loads(ld.text) + if data['@type'].lower() == 'recipe': + return { 'raw': data, 'recipe': await get_recipe_from_ldata(conn, url, data) } + except (json.decoder.JSONDecodeError, KeyError): + pass + + return None \ No newline at end of file