munch-ease-backend/products/__init__.py

import json

from products.db import Product, find_product_by_tag, find_product_by_product_id, insert_product, get_tags, add_tag, find_product_by_id
from products.scraping import scrape_woolies_data, get_product_id, get_product_details_url

from typing import List
import re

def get_package_size(data: dict) -> str:
    size = data['Product']['PackageSize']
    if size:
        match = re.match(r'(\d+)(.*)', size)
        if match:
            return int(match.group(1)), match.group(2)

    return 1, 'items'

async def create_product(link: str) -> Product:
    product_id = get_product_id(link)
    if not product_id:
        return None, None
    
    product_url = get_product_details_url(product_id)
    product, data = None, await scrape_woolies_data(product_url)
    if data:
        _dump_json_data_to_log(data, product_id)

        quantity, unit = get_package_size(data)
        product =  Product(
            id=0,
            product_id=product_id,
            name=data['Product']['Name'],
            link=link,
            quantity=quantity,
            unit=unit,
            img_small=data['Product']['SmallImageFile'],
            img_large=data['Product']['LargeImageFile'],
        )

    return product, data
    
async def add_missing_tags(conn, product: Product, tags: List[str]):
    existing_tags = set()
    async for tag in get_tags(conn, product):
        existing_tags.add(tag)
    
    remaining_tags = set(tags) - existing_tags
    if not remaining_tags:
        return False
    
    for tag in remaining_tags:
        await add_tag(conn, product, tag)
    
    return product

async def get_or_create(conn, url: str, tags: List[str]) -> Product:
    product_id = get_product_id(url)
    if not product_id:
        return None
    
    existing = await find_product_by_product_id(conn, product_id)
    if existing:
        await add_missing_tags(conn, existing, tags)
        return existing
    
    product, data = await create_product(url)
    if product:
        await insert_product(conn, product, data)
        await add_missing_tags(conn, product, tags)

    return product

def _dump_json_data_to_log(data: dict, product_id: str) -> str:
    import os, re
    dir = './data/dump'
    if not os.path.exists(dir):
        os.makedirs(dir)

    prefix = f'product_{product_id}'
    suffix = '.json'
    file_ids = [int(re.findall(r'\d+', f)[0]) for f in os.listdir(dir) if re.match(prefix + r'\d+' + suffix, f)]
    id = max(file_ids) + 1 if file_ids else 0
    filename = f'{prefix}{id}{suffix}'
    with open(os.path.join(dir, filename), 'w') as f:
        json.dump(data, f, indent=4)