Moved Ingredient to own namespace

This commit is contained in:
jableader 2024-01-17 18:21:16 +11:00
parent 77ce3658bf
commit c8b844d3e8
8 changed files with 144 additions and 116 deletions

3
db.py
View file

@ -8,6 +8,9 @@ async def create():
conn = await connect() conn = await connect()
await product_db.create(conn) await product_db.create(conn)
import ingredients.db as ingredient_db
await ingredient_db.create(conn)
import recipes.db as recipe_db import recipes.db as recipe_db
await recipe_db.create(conn) await recipe_db.create(conn)

60
ingredients/__init__.py Normal file
View file

@ -0,0 +1,60 @@
from ingredients.db import Ingredient, find_ingredients_by_meal_id, find_ingredients_by_recipe_id
import units
from products import Product, find_product_by_tag
from ingredient_parser import parse_multiple_ingredients
from typing import List
def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient:
results = []
for ingredient in parse_multiple_ingredients(ingredients):
name = ingredient.name.text if ingredient.name else ''
quantity, unit = None, None
for amount in ingredient.amount:
if quantity is None and amount.quantity:
quantity = amount.quantity
if unit is None and amount.unit:
real_unit = units.get_unit(amount.unit)
if real_unit:
unit = real_unit.name
if isinstance(quantity, str):
try:
quantity = float(quantity)
except ValueError:
pass
if quantity is None or not isinstance(quantity, (int, float)):
quantity = 1
if unit is None:
unit = units.ITEMS.name
results.append(Ingredient(id=0,
line=ingredient.sentence,
name=name,
quantity=quantity,
unit=unit,
preparation=ingredient.preparation.text if ingredient.preparation else '',
product_id=0
))
return results
async def _find_existing_product(conn, ingredient: str) -> Product:
async for item in find_product_by_tag(conn, ingredient):
return item
return None
async def match_existing_products(conn, ingredients: List[Ingredient]) -> List[Ingredient]:
for ingredient in ingredients:
if not ingredient.product:
existing = await _find_existing_product(conn, ingredient.name)
if existing:
ingredient.product_id = existing.id
ingredient.product = existing
return ingredients

58
ingredients/db.py Normal file
View file

@ -0,0 +1,58 @@
from products import Product
from pydantic import BaseModel
from typing import List, ClassVar, Optional
class Ingredient(BaseModel):
KEYS: ClassVar[List[str]] = ['id', 'name', 'line', 'preparation', 'unit', 'quantity', 'product_id', 'recipe_id', 'meal_id']
id: int
name: str
line: str
unit: str
quantity: float
preparation: str
product_id: Optional[int] = None
recipe_id: Optional[int] = None
meal_id: Optional[int] = None
product: Product = None
async def create(conn):
await conn.execute('''
CREATE TABLE IF NOT EXISTS Ingredient (
id INTEGER PRIMARY KEY,
name TEXT,
line TEXT,
preparation TEXT,
unit TEXT,
quantity REAL,
product_id INTEGER,
recipe_id INTEGER,
meal_id INTEGER,
FOREIGN KEY (product_id) REFERENCES Product(id),
FOREIGN KEY (recipe_id) REFERENCES Recipe(id),
FOREIGN KEY (meal_id) REFERENCES Meal(id)
);''')
async def insert_ingredient(conn, ingredient: Ingredient):
async with conn.execute('''
INSERT INTO Ingredient (name, line, preparation, unit, quantity, product_id, recipe_id, meal_id)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (ingredient.name, ingredient.line, ingredient.preparation, ingredient.unit, ingredient.quantity, ingredient.product_id, ingredient.recipe_id, ingredient.meal_id)) as cursor:
ingredient.id = cursor.lastrowid
async def find_ingredients_by_recipe_id(conn, recipe_id: int) -> List[Ingredient]:
async with conn.execute(f'''
SELECT {','.join(Ingredient.KEYS)} FROM Ingredient
WHERE recipe_id = ?
''', (recipe_id,)) as cursor:
async for row in cursor:
yield Ingredient(**{k:v for k,v in zip(Ingredient.KEYS, row)})
async def find_ingredients_by_meal_id(conn, meal_id: int) -> List[Ingredient]:
async with conn.execute(f'''
SELECT {','.join(Ingredient.KEYS)} FROM Ingredient
WHERE meal_id = ?
''', (meal_id,)) as cursor:
async for row in cursor:
yield Ingredient(**{k:v for k,v in zip(Ingredient.KEYS, row)})

11
main.py
View file

@ -1,5 +1,5 @@
import sqlite3 import sqlite3
import products, recipes, db, meals, persons import products, recipes, db, meals, persons, ingredients
import datetime import datetime
from pydantic import BaseModel from pydantic import BaseModel
@ -39,7 +39,7 @@ async def parse_ingredients(lines: Annotated[
List[str], List[str],
Query(alias="ingredients", Query(alias="ingredients",
title="Array of ingredients to parse")], title="Array of ingredients to parse")],
conn: sqlite3.Connection = Depends(get_db)) -> List[recipes.Ingredient]: conn: sqlite3.Connection = Depends(get_db)) -> List[ingredients.Ingredient]:
ingredients = recipes.parse_ingredient_from_nlp(lines) ingredients = recipes.parse_ingredient_from_nlp(lines)
await recipes.match_existing_products(conn, ingredients) await recipes.match_existing_products(conn, ingredients)
return ingredients return ingredients
@ -58,7 +58,7 @@ async def load_full_recipe(conn: sqlite3.Connection, id: int) -> recipes.Recipe:
return None return None
r.ingredients = [] r.ingredients = []
async for ingredient in recipes.find_ingredients_by_recipe_id(conn, id): async for ingredient in ingredients.find_ingredients_by_recipe_id(conn, id):
ingredient.product = await products.find_product_by_id(conn, ingredient.product_id) ingredient.product = await products.find_product_by_id(conn, ingredient.product_id)
r.ingredients.append(ingredient) r.ingredients.append(ingredient)
@ -76,7 +76,7 @@ async def get_recipes(q: str | None = None, conn: sqlite3.Connection = Depends(g
for recipe in result: for recipe in result:
recipe.ingredients = [] recipe.ingredients = []
async for ingredient in recipes.find_ingredients_by_recipe_id(conn, recipe.id): async for ingredient in ingredients.find_ingredients_by_recipe_id(conn, recipe.id):
ingredient.product = await products.find_product_by_id(conn, ingredient.product_id) ingredient.product = await products.find_product_by_id(conn, ingredient.product_id)
recipe.ingredients.append(ingredient) recipe.ingredients.append(ingredient)
@ -128,6 +128,9 @@ async def create_meal(meal: meals.Meal, conn: sqlite3.Connection = Depends(get_d
if not meal.consumers: if not meal.consumers:
return JSONResponse(status_code=400, content={'message': 'Meal must have at least one consumer'}) return JSONResponse(status_code=400, content={'message': 'Meal must have at least one consumer'})
if len(meal.recipes) == 0 and len(meal.extra_ingredients) == 0:
return JSONResponse(status_code=400, content={'message': 'Meal must have at least one recipe or ingredient'})
await meals.insert_meal(conn, meal) await meals.insert_meal(conn, meal)
await conn.commit() await conn.commit()
return meal return meal

View file

@ -1,6 +1,7 @@
from typing import List, ClassVar from typing import List, ClassVar
from pydantic import BaseModel from pydantic import BaseModel
from persons import Person from persons import Person
from ingredients import Ingredient
import datetime import datetime
@ -12,6 +13,7 @@ class Meal(BaseModel):
cleanup: List[Person] = [] cleanup: List[Person] = []
consumers: List[Person] = [] consumers: List[Person] = []
recipes: List[Person] = [] recipes: List[Person] = []
extra_ingredients: List[Ingredient] = []
async def create(conn): async def create(conn):
await conn.execute(''' await conn.execute('''

View file

@ -1,68 +1,18 @@
from products import Product, find_product_by_tag from recipes.db import Recipe, insert_recipe, find_recipe_by_id, get_all, find_recipes_by_name
from recipes.db import Recipe, Ingredient, insert_recipe, insert_ingredient, find_recipe_by_id, find_ingredients_by_recipe_id, get_all, find_recipes_by_name from recipes.scraping import scrape_recipe_ldata as _scrape_recipe_ldata
from recipes.scraping import scrape_recipe from ingredients import parse_ingredient_from_nlp as _parse_ingredient_from_nlp, match_existing_products as _match_existing_products
from ingredient_parser import parse_multiple_ingredients import json
from typing import List
import json, units async def parse_recipe(conn, url: str) -> Recipe:
ldata = await _scrape_recipe_ldata(url)
async def find_existing_product(conn, ingredient: str) -> Product: if ldata:
async for item in find_product_by_tag(conn, ingredient): return await _get_recipe_from_ldata(conn, url, ldata)
return item
return None return None
def parse_ingredient_from_nlp(ingredients: List[str]) -> Ingredient:
results = []
for ingredient in parse_multiple_ingredients(ingredients):
name = ingredient.name.text if ingredient.name else ''
quantity, unit = None, None
for amount in ingredient.amount:
if quantity is None and amount.quantity:
quantity = amount.quantity
if unit is None and amount.unit:
real_unit = units.get_unit(amount.unit)
if real_unit:
unit = real_unit.name
if isinstance(quantity, str):
try:
quantity = float(quantity)
except ValueError:
pass
if quantity is None or not isinstance(quantity, (int, float)):
quantity = 1
if unit is None:
unit = units.ITEMS.name
results.append(Ingredient(id=0,
line=ingredient.sentence,
name=name,
quantity=quantity,
unit=unit,
preparation=ingredient.preparation.text if ingredient.preparation else '',
product_id=0
))
return results
async def match_existing_products(conn, ingredients: List[Ingredient]) -> List[Ingredient]:
for ingredient in ingredients:
if not ingredient.product:
existing = await find_existing_product(conn, ingredient.name)
if existing:
ingredient.product_id = existing.id
ingredient.product = existing
return ingredients
async def _get_recipe_from_ldata(conn, url: str, ldata: dict) -> dict: async def _get_recipe_from_ldata(conn, url: str, ldata: dict) -> dict:
ingredients = parse_ingredient_from_nlp(ldata['recipeIngredient']) ingredients = _parse_ingredient_from_nlp(ldata['recipeIngredient'])
ingredients = await match_existing_products(conn, ingredients) ingredients = await _match_existing_products(conn, ingredients)
name = ldata['name'] if 'name' in ldata else url name = ldata['name'] if 'name' in ldata else url
images = ldata['image'] if 'image' in ldata else [] images = ldata['image'] if 'image' in ldata else []
@ -82,10 +32,4 @@ async def _get_recipe_from_ldata(conn, url: str, ldata: dict) -> dict:
image_urls=images, image_urls=images,
raw_data=json.dumps(ldata), raw_data=json.dumps(ldata),
ingredients=ingredients ingredients=ingredients
) )
async def parse_recipe(conn, url: str) -> dict:
ldata = await scrape_recipe(url)
if ldata:
return await _get_recipe_from_ldata(conn, url, ldata)
return None

View file

@ -1,22 +1,9 @@
import json import json
import aiosqlite
from products import Product from ingredients import Ingredient
from pydantic import BaseModel from pydantic import BaseModel
from typing import List, ClassVar, Optional from typing import List, ClassVar
class Ingredient(BaseModel):
KEYS: ClassVar[List[str]] = ['id', 'name', 'line', 'preparation', 'unit', 'quantity', 'product_id', 'recipe_id']
id: int
name: str
line: str
unit: str
quantity: float
preparation: str
product_id: Optional[int] = None
recipe_id: Optional[int] = None
product: Product = None
class Recipe(BaseModel): class Recipe(BaseModel):
KEYS: ClassVar[List[str]] = ['id', 'name', 'link', 'image_urls', 'raw_data'] KEYS: ClassVar[List[str]] = ['id', 'name', 'link', 'image_urls', 'raw_data']
@ -37,27 +24,6 @@ async def create(conn):
raw_data TEXT raw_data TEXT
);''') );''')
await conn.execute('''
CREATE TABLE IF NOT EXISTS Ingredient (
id INTEGER PRIMARY KEY,
name TEXT,
line TEXT,
preparation TEXT,
unit TEXT,
quantity REAL,
product_id INTEGER,
recipe_id INTEGER,
FOREIGN KEY (product_id) REFERENCES Product(id),
FOREIGN KEY (recipe_id) REFERENCES Recipe(id)
);''')
async def insert_ingredient(conn, ingredient: Ingredient):
async with conn.execute('''
INSERT INTO Ingredient (name, line, preparation, unit, quantity, product_id, recipe_id)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (ingredient.name, ingredient.line, ingredient.preparation, ingredient.unit, ingredient.quantity, ingredient.product_id, ingredient.recipe_id)) as cursor:
ingredient.id = cursor.lastrowid
async def insert_recipe(conn, recipe: Recipe): async def insert_recipe(conn, recipe: Recipe):
async with conn.execute(''' async with conn.execute('''
INSERT INTO Recipe (name, link, raw_data, image_urls) INSERT INTO Recipe (name, link, raw_data, image_urls)
@ -78,14 +44,6 @@ async def find_recipe_by_id(conn, recipe_id: int) -> Recipe:
''', (recipe_id,)) as cursor: ''', (recipe_id,)) as cursor:
async for row in cursor: async for row in cursor:
return row_to_recipe(row) return row_to_recipe(row)
async def find_ingredients_by_recipe_id(conn, recipe_id: int) -> List[Ingredient]:
async with conn.execute(f'''
SELECT {','.join(Ingredient.KEYS)} FROM Ingredient
WHERE recipe_id = ?
''', (recipe_id,)) as cursor:
async for row in cursor:
yield Ingredient(**{k:v for k,v in zip(Ingredient.KEYS, row)})
async def find_recipes_by_name(conn, name: str) -> List[Recipe]: async def find_recipes_by_name(conn, name: str) -> List[Recipe]:
async with conn.execute(f''' async with conn.execute(f'''

View file

@ -13,7 +13,7 @@ def _is_recipe_ldata(ldata_node):
return None return None
async def scrape_recipe(url: str) -> dict: async def scrape_recipe_ldata(url: str) -> dict:
headers = { headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
"Accept-Language": "en-US,en;q=0.9", "Accept-Language": "en-US,en;q=0.9",