From c8b844d3e8c70957fddbfe6fb80041c99f3c0ee0 Mon Sep 17 00:00:00 2001 From: jableader Date: Wed, 17 Jan 2024 18:21:16 +1100 Subject: [PATCH] Moved Ingredient to own namespace --- db.py | 3 ++ ingredients/__init__.py | 60 +++++++++++++++++++++++++++++++ ingredients/db.py | 58 ++++++++++++++++++++++++++++++ main.py | 11 +++--- meals/db.py | 2 ++ recipes/__init__.py | 78 ++++++----------------------------------- recipes/db.py | 46 ++---------------------- recipes/scraping.py | 2 +- 8 files changed, 144 insertions(+), 116 deletions(-) create mode 100644 ingredients/__init__.py create mode 100644 ingredients/db.py diff --git a/db.py b/db.py index a7188e7..fffb900 100644 --- a/db.py +++ b/db.py @@ -8,6 +8,9 @@ async def create(): conn = await connect() await product_db.create(conn) + import ingredients.db as ingredient_db + await ingredient_db.create(conn) + import recipes.db as recipe_db await recipe_db.create(conn) diff --git a/ingredients/__init__.py b/ingredients/__init__.py new file mode 100644 index 0000000..e7547a2 --- /dev/null +++ b/ingredients/__init__.py @@ -0,0 +1,60 @@ +from ingredients.db import Ingredient, find_ingredients_by_meal_id, find_ingredients_by_recipe_id + +import units +from products import Product, find_product_by_tag + +from ingredient_parser import parse_multiple_ingredients +from typing import List + +def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient: + results = [] + for ingredient in parse_multiple_ingredients(ingredients): + name = ingredient.name.text if ingredient.name else '' + + quantity, unit = None, None + for amount in ingredient.amount: + if quantity is None and amount.quantity: + quantity = amount.quantity + + if unit is None and amount.unit: + real_unit = units.get_unit(amount.unit) + if real_unit: + unit = real_unit.name + + if isinstance(quantity, str): + try: + quantity = float(quantity) + except ValueError: + pass + + if quantity is None or not isinstance(quantity, (int, float)): + quantity = 1 + + if unit is None: + unit = units.ITEMS.name + + results.append(Ingredient(id=0, + line=ingredient.sentence, + name=name, + quantity=quantity, + unit=unit, + preparation=ingredient.preparation.text if ingredient.preparation else '', + product_id=0 + )) + + return results + +async def _find_existing_product(conn, ingredient: str) -> Product: + async for item in find_product_by_tag(conn, ingredient): + return item + return None + +async def match_existing_products(conn, ingredients: List[Ingredient]) -> List[Ingredient]: + for ingredient in ingredients: + if not ingredient.product: + existing = await _find_existing_product(conn, ingredient.name) + if existing: + ingredient.product_id = existing.id + ingredient.product = existing + + return ingredients \ No newline at end of file diff --git a/ingredients/db.py b/ingredients/db.py new file mode 100644 index 0000000..8875d5b --- /dev/null +++ b/ingredients/db.py @@ -0,0 +1,58 @@ +from products import Product + +from pydantic import BaseModel +from typing import List, ClassVar, Optional + +class Ingredient(BaseModel): + KEYS: ClassVar[List[str]] = ['id', 'name', 'line', 'preparation', 'unit', 'quantity', 'product_id', 'recipe_id', 'meal_id'] + id: int + name: str + line: str + unit: str + quantity: float + preparation: str + product_id: Optional[int] = None + recipe_id: Optional[int] = None + meal_id: Optional[int] = None + product: Product = None + +async def create(conn): + await conn.execute(''' + CREATE TABLE IF NOT EXISTS Ingredient ( + id INTEGER PRIMARY KEY, + name TEXT, + line TEXT, + preparation TEXT, + unit TEXT, + quantity REAL, + product_id INTEGER, + recipe_id INTEGER, + meal_id INTEGER, + FOREIGN KEY (product_id) REFERENCES Product(id), + FOREIGN KEY (recipe_id) REFERENCES Recipe(id), + FOREIGN KEY (meal_id) REFERENCES Meal(id) + );''') + + +async def insert_ingredient(conn, ingredient: Ingredient): + async with conn.execute(''' + INSERT INTO Ingredient (name, line, preparation, unit, quantity, product_id, recipe_id, meal_id) + VALUES (?, ?, ?, ?, ?, ?, ?) + ''', (ingredient.name, ingredient.line, ingredient.preparation, ingredient.unit, ingredient.quantity, ingredient.product_id, ingredient.recipe_id, ingredient.meal_id)) as cursor: + ingredient.id = cursor.lastrowid + +async def find_ingredients_by_recipe_id(conn, recipe_id: int) -> List[Ingredient]: + async with conn.execute(f''' + SELECT {','.join(Ingredient.KEYS)} FROM Ingredient + WHERE recipe_id = ? + ''', (recipe_id,)) as cursor: + async for row in cursor: + yield Ingredient(**{k:v for k,v in zip(Ingredient.KEYS, row)}) + +async def find_ingredients_by_meal_id(conn, meal_id: int) -> List[Ingredient]: + async with conn.execute(f''' + SELECT {','.join(Ingredient.KEYS)} FROM Ingredient + WHERE meal_id = ? + ''', (meal_id,)) as cursor: + async for row in cursor: + yield Ingredient(**{k:v for k,v in zip(Ingredient.KEYS, row)}) \ No newline at end of file diff --git a/main.py b/main.py index 516f8ca..e76d1aa 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ import sqlite3 -import products, recipes, db, meals, persons +import products, recipes, db, meals, persons, ingredients import datetime from pydantic import BaseModel @@ -39,7 +39,7 @@ async def parse_ingredients(lines: Annotated[ List[str], Query(alias="ingredients", title="Array of ingredients to parse")], - conn: sqlite3.Connection = Depends(get_db)) -> List[recipes.Ingredient]: + conn: sqlite3.Connection = Depends(get_db)) -> List[ingredients.Ingredient]: ingredients = recipes.parse_ingredient_from_nlp(lines) await recipes.match_existing_products(conn, ingredients) return ingredients @@ -58,7 +58,7 @@ async def load_full_recipe(conn: sqlite3.Connection, id: int) -> recipes.Recipe: return None r.ingredients = [] - async for ingredient in recipes.find_ingredients_by_recipe_id(conn, id): + async for ingredient in ingredients.find_ingredients_by_recipe_id(conn, id): ingredient.product = await products.find_product_by_id(conn, ingredient.product_id) r.ingredients.append(ingredient) @@ -76,7 +76,7 @@ async def get_recipes(q: str | None = None, conn: sqlite3.Connection = Depends(g for recipe in result: recipe.ingredients = [] - async for ingredient in recipes.find_ingredients_by_recipe_id(conn, recipe.id): + async for ingredient in ingredients.find_ingredients_by_recipe_id(conn, recipe.id): ingredient.product = await products.find_product_by_id(conn, ingredient.product_id) recipe.ingredients.append(ingredient) @@ -128,6 +128,9 @@ async def create_meal(meal: meals.Meal, conn: sqlite3.Connection = Depends(get_d if not meal.consumers: return JSONResponse(status_code=400, content={'message': 'Meal must have at least one consumer'}) + if len(meal.recipes) == 0 and len(meal.extra_ingredients) == 0: + return JSONResponse(status_code=400, content={'message': 'Meal must have at least one recipe or ingredient'}) + await meals.insert_meal(conn, meal) await conn.commit() return meal diff --git a/meals/db.py b/meals/db.py index a00353a..b63fa70 100644 --- a/meals/db.py +++ b/meals/db.py @@ -1,6 +1,7 @@ from typing import List, ClassVar from pydantic import BaseModel from persons import Person +from ingredients import Ingredient import datetime @@ -12,6 +13,7 @@ class Meal(BaseModel): cleanup: List[Person] = [] consumers: List[Person] = [] recipes: List[Person] = [] + extra_ingredients: List[Ingredient] = [] async def create(conn): await conn.execute(''' diff --git a/recipes/__init__.py b/recipes/__init__.py index 2ff39bf..ba5ee32 100644 --- a/recipes/__init__.py +++ b/recipes/__init__.py @@ -1,68 +1,18 @@ -from products import Product, find_product_by_tag -from recipes.db import Recipe, Ingredient, insert_recipe, insert_ingredient, find_recipe_by_id, find_ingredients_by_recipe_id, get_all, find_recipes_by_name -from recipes.scraping import scrape_recipe +from recipes.db import Recipe, insert_recipe, find_recipe_by_id, get_all, find_recipes_by_name +from recipes.scraping import scrape_recipe_ldata as _scrape_recipe_ldata +from ingredients import parse_ingredient_from_nlp as _parse_ingredient_from_nlp, match_existing_products as _match_existing_products -from ingredient_parser import parse_multiple_ingredients -from typing import List +import json -import json, units - -async def find_existing_product(conn, ingredient: str) -> Product: - async for item in find_product_by_tag(conn, ingredient): - return item +async def parse_recipe(conn, url: str) -> Recipe: + ldata = await _scrape_recipe_ldata(url) + if ldata: + return await _get_recipe_from_ldata(conn, url, ldata) return None - -def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient: - results = [] - for ingredient in parse_multiple_ingredients(ingredients): - name = ingredient.name.text if ingredient.name else '' - - quantity, unit = None, None - for amount in ingredient.amount: - if quantity is None and amount.quantity: - quantity = amount.quantity - - if unit is None and amount.unit: - real_unit = units.get_unit(amount.unit) - if real_unit: - unit = real_unit.name - - if isinstance(quantity, str): - try: - quantity = float(quantity) - except ValueError: - pass - - if quantity is None or not isinstance(quantity, (int, float)): - quantity = 1 - - if unit is None: - unit = units.ITEMS.name - - results.append(Ingredient(id=0, - line=ingredient.sentence, - name=name, - quantity=quantity, - unit=unit, - preparation=ingredient.preparation.text if ingredient.preparation else '', - product_id=0 - )) - return results - -async def match_existing_products(conn, ingredients: List[Ingredient]) -> List[Ingredient]: - for ingredient in ingredients: - if not ingredient.product: - existing = await find_existing_product(conn, ingredient.name) - if existing: - ingredient.product_id = existing.id - ingredient.product = existing - - return ingredients - async def _get_recipe_from_ldata(conn, url: str, ldata: dict) -> dict: - ingredients = parse_ingredient_from_nlp(ldata['recipeIngredient']) - ingredients = await match_existing_products(conn, ingredients) + ingredients = _parse_ingredient_from_nlp(ldata['recipeIngredient']) + ingredients = await _match_existing_products(conn, ingredients) name = ldata['name'] if 'name' in ldata else url images = ldata['image'] if 'image' in ldata else [] @@ -82,10 +32,4 @@ async def _get_recipe_from_ldata(conn, url: str, ldata: dict) -> dict: image_urls=images, raw_data=json.dumps(ldata), ingredients=ingredients - ) - -async def parse_recipe(conn, url: str) -> dict: - ldata = await scrape_recipe(url) - if ldata: - return await _get_recipe_from_ldata(conn, url, ldata) - return None \ No newline at end of file + ) \ No newline at end of file diff --git a/recipes/db.py b/recipes/db.py index c59965f..f844045 100644 --- a/recipes/db.py +++ b/recipes/db.py @@ -1,22 +1,9 @@ import json -import aiosqlite -from products import Product +from ingredients import Ingredient from pydantic import BaseModel -from typing import List, ClassVar, Optional - -class Ingredient(BaseModel): - KEYS: ClassVar[List[str]] = ['id', 'name', 'line', 'preparation', 'unit', 'quantity', 'product_id', 'recipe_id'] - id: int - name: str - line: str - unit: str - quantity: float - preparation: str - product_id: Optional[int] = None - recipe_id: Optional[int] = None - product: Product = None +from typing import List, ClassVar class Recipe(BaseModel): KEYS: ClassVar[List[str]] = ['id', 'name', 'link', 'image_urls', 'raw_data'] @@ -37,27 +24,6 @@ async def create(conn): raw_data TEXT );''') - await conn.execute(''' - CREATE TABLE IF NOT EXISTS Ingredient ( - id INTEGER PRIMARY KEY, - name TEXT, - line TEXT, - preparation TEXT, - unit TEXT, - quantity REAL, - product_id INTEGER, - recipe_id INTEGER, - FOREIGN KEY (product_id) REFERENCES Product(id), - FOREIGN KEY (recipe_id) REFERENCES Recipe(id) - );''') - -async def insert_ingredient(conn, ingredient: Ingredient): - async with conn.execute(''' - INSERT INTO Ingredient (name, line, preparation, unit, quantity, product_id, recipe_id) - VALUES (?, ?, ?, ?, ?, ?, ?) - ''', (ingredient.name, ingredient.line, ingredient.preparation, ingredient.unit, ingredient.quantity, ingredient.product_id, ingredient.recipe_id)) as cursor: - ingredient.id = cursor.lastrowid - async def insert_recipe(conn, recipe: Recipe): async with conn.execute(''' INSERT INTO Recipe (name, link, raw_data, image_urls) @@ -78,14 +44,6 @@ async def find_recipe_by_id(conn, recipe_id: int) -> Recipe: ''', (recipe_id,)) as cursor: async for row in cursor: return row_to_recipe(row) - -async def find_ingredients_by_recipe_id(conn, recipe_id: int) -> List[Ingredient]: - async with conn.execute(f''' - SELECT {','.join(Ingredient.KEYS)} FROM Ingredient - WHERE recipe_id = ? - ''', (recipe_id,)) as cursor: - async for row in cursor: - yield Ingredient(**{k:v for k,v in zip(Ingredient.KEYS, row)}) async def find_recipes_by_name(conn, name: str) -> List[Recipe]: async with conn.execute(f''' diff --git a/recipes/scraping.py b/recipes/scraping.py index 06ea219..38fe5cb 100644 --- a/recipes/scraping.py +++ b/recipes/scraping.py @@ -13,7 +13,7 @@ def _is_recipe_ldata(ldata_node): return None -async def scrape_recipe(url: str) -> dict: +async def scrape_recipe_ldata(url: str) -> dict: headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", "Accept-Language": "en-US,en;q=0.9",