Logged query
+${escapeHtml(item.query)}
+${escapeHtml(status)}
+Logged: ${escapeHtml(item.created_at || '')}
+Current matching recipes: ${escapeHtml(item.result_count || 0)}
+- ${titles || '
- No recovered recipe titles yet. '}
diff --git a/automation/bin/kitchenowl_recipe_import.py b/automation/bin/kitchenowl_recipe_import.py index fc7b67a..1973f10 100755 --- a/automation/bin/kitchenowl_recipe_import.py +++ b/automation/bin/kitchenowl_recipe_import.py @@ -236,12 +236,118 @@ def coerce_text(value: Any) -> str: return unescape(str(value)).strip() +def normalize_whitespace(value: str) -> str: + cleaned = re.sub(r'\s+', ' ', value or '').strip() + return re.sub(r'(?<=[.!?])(?=[A-Z])', ' ', cleaned) + + +def split_paragraphs(value: str) -> list[str]: + parts: list[str] = [] + for chunk in re.split(r'\n\s*\n|\r\n\r\n', value or ''): + cleaned = normalize_whitespace(chunk) + if cleaned: + parts.append(cleaned) + return parts + + +def extract_instruction_steps(value: Any) -> list[str]: + steps: list[str] = [] + + def visit(node: Any) -> None: + if node is None: + return + if isinstance(node, str): + for chunk in re.split(r'\n+', node): + cleaned = normalize_whitespace(chunk) + if cleaned: + steps.append(cleaned) + return + if isinstance(node, list): + for item in node: + visit(item) + return + if isinstance(node, dict): + if node.get('@type') == 'HowToSection': + name = normalize_whitespace(coerce_text(node.get('name'))) + if name: + steps.append(f'{name}:') + visit(node.get('itemListElement')) + return + if node.get('@type') in {'HowToStep', 'ListItem'}: + text = normalize_whitespace(coerce_text(node.get('text') or node.get('name') or node.get('item'))) + if text: + steps.append(text) + elif node.get('itemListElement') is not None: + visit(node.get('itemListElement')) + return + if node.get('itemListElement') is not None: + visit(node.get('itemListElement')) + return + text = normalize_whitespace(coerce_text(node.get('text') or node.get('name'))) + if text: + steps.append(text) + + visit(value) + deduped: list[str] = [] + seen: set[str] = set() + for step in steps: + key = step.lower() + if key in seen: + continue + seen.add(key) + deduped.append(step) + return deduped + + +def build_description(recipe_node: dict[str, Any]) -> str: + intro_parts = split_paragraphs(coerce_text(recipe_node.get('description'))) + intro = intro_parts[0] if intro_parts else '' + notes = intro_parts[1:] if len(intro_parts) > 1 else [] + steps = extract_instruction_steps(recipe_node.get('recipeInstructions')) + parts: list[str] = [] + if intro: + parts.append(intro) + if steps: + rendered_steps = [] + step_number = 1 + for step in steps: + if step.endswith(':'): + rendered_steps.append(step) + else: + rendered_steps.append(f'{step_number}. {step}') + step_number += 1 + parts.append('\n'.join(rendered_steps)) + elif intro_parts[1:]: + parts.append('\n'.join(intro_parts[1:])) + if notes: + parts.append('Notes:\n- ' + '\n- '.join(notes)) + return '\n\n'.join(part for part in parts if part).strip() + + def slugify_tag(value: str) -> str: value = value.strip().lower() value = re.sub(r'[^a-z0-9]+', '-', value) return value.strip('-')[:48] +def extract_image_url(value: Any) -> str | None: + if isinstance(value, str): + cleaned = value.strip() + return cleaned or None + if isinstance(value, list): + for item in value: + found = extract_image_url(item) + if found: + return found + return None + if isinstance(value, dict): + for key in ('url', 'contentUrl'): + found = extract_image_url(value.get(key)) + if found: + return found + return None + + def normalize_keywords(value: Any) -> list[str]: out: list[str] = [] if isinstance(value, str): @@ -294,15 +400,7 @@ def split_ingredient(line: str) -> tuple[str, str]: def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str | None = None) -> dict[str, Any]: name = coerce_text(recipe_node.get('name')) or (title_fallback or url) - description_parts: list[str] = [] - desc = coerce_text(recipe_node.get('description')) - instructions = recipe_node.get('recipeInstructions') - instructions_text = coerce_text(instructions) - if desc: - description_parts.append(desc) - if instructions_text: - description_parts.append(instructions_text) - description = '\n\n'.join(part for part in description_parts if part).strip() + description = build_description(recipe_node) ingredients = recipe_node.get('recipeIngredient') or [] items = [] @@ -348,6 +446,9 @@ def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str 'items': items, 'tags': deduped_tags[:20], } + image_url = extract_image_url(recipe_node.get('image')) + if image_url: + payload['photo'] = image_url cook_time = parse_duration_minutes(recipe_node.get('cookTime')) prep_time = parse_duration_minutes(recipe_node.get('prepTime')) total_time = parse_duration_minutes(recipe_node.get('totalTime')) diff --git a/docs/operations/KITCHENOWL_RECIPE_IMPORT.md b/docs/operations/KITCHENOWL_RECIPE_IMPORT.md index 009d1ea..6c298c4 100644 --- a/docs/operations/KITCHENOWL_RECIPE_IMPORT.md +++ b/docs/operations/KITCHENOWL_RECIPE_IMPORT.md @@ -122,7 +122,7 @@ It pulls: - Some sites block scraping aggressively; Doris first tries normal fetch, then falls back to `curl` headers for better odds. - Ingredient normalization is heuristic, not magic. It aims for a sane KitchenOwl item name plus a description/amount, but some weird ingredients may still need manual cleanup. -- Image upload/import is not wired yet in this helper. +- Recipe source images are now forwarded into KitchenOwl when the source exposes a usable image URL. - The helper expects structured JSON-LD. If a site hides the recipe entirely in client-side blobs or anti-bot nonsense, manual copy/paste may still be needed. - `GET /api/settings` on the live KitchenOwl instance currently returns `500`, but that does not block recipe import. diff --git a/home/doris-kitchen/app/routes/api_dashboard.py b/home/doris-kitchen/app/routes/api_dashboard.py index 617b598..73ede2f 100644 --- a/home/doris-kitchen/app/routes/api_dashboard.py +++ b/home/doris-kitchen/app/routes/api_dashboard.py @@ -1,9 +1,13 @@ from fastapi import APIRouter +from fastapi import HTTPException from app.services.repository import repo +from app.services.recipe_search import recipe_search +from app.services.store import JsonStore router = APIRouter(prefix="/api", tags=["dashboard"]) +store = JsonStore() @router.get("/health") @@ -16,12 +20,14 @@ async def summary(): profile = repo.get_profile() suggestions = repo.list_suggestions() plans = repo.list_meal_plans() + failed_searches = repo.recent_failed_searches(limit=6) counts = { "suggested": sum(1 for item in suggestions if item.get("status") == "suggested"), "approved": sum(1 for item in suggestions if item.get("status") == "approved"), "saved": sum(1 for item in suggestions if item.get("status") == "saved"), "rejected": sum(1 for item in suggestions if item.get("status") == "rejected"), "plans": len(plans), + "failed_searches": len(repo.list_failed_searches()), } return { "backend": repo.backend, @@ -29,5 +35,65 @@ async def summary(): "counts": counts, "recent_suggestions": repo.recent_suggestions(), "recent_plans": list(reversed(plans[-4:])), + "recent_failed_searches": failed_searches, } + +@router.get("/failed-searches") +async def failed_searches(): + return {"status": "ok", "items": repo.list_failed_searches()} + + +@router.get("/failed-searches/{failed_search_id}/results") +async def failed_search_results(failed_search_id: str): + item = next((entry for entry in repo.list_failed_searches() if str(entry.get("id")) == str(failed_search_id)), None) + if not item: + raise HTTPException(status_code=404, detail="Failed search not found.") + query = str(item.get("query") or "").strip() + if not query: + raise HTTPException(status_code=400, detail="Failed search query is empty.") + all_results = recipe_search.search(query, limit=10) + results = [entry for entry in all_results if str(entry.get("status") or "suggested") == "suggested"] + updates = { + "last_checked_at": store.now(), + "updated_at": store.now(), + "result_count": len(results), + "sample_titles": [str(entry.get("title") or "") for entry in results[:4]], + } + if results: + updates["status"] = "resolved-with-recipes" + updates["resolved_at"] = str(item.get("resolved_at") or store.now()) + saved = repo.update_failed_search(str(item.get("id")), updates) or {**item, **updates} + return {"status": "ok", "item": saved, "results": results} + + +@router.post("/failed-searches/refresh") +async def refresh_failed_searches(): + items = repo.list_failed_searches() + refreshed = [] + for item in items: + query = str(item.get("query") or "").strip() + status = str(item.get("status") or "open") + if not query or status != "open": + refreshed.append(item) + continue + preview_titles: list[str] = [] + result_count = 0 + for candidate_query in recipe_search._candidate_queries(query): + hits = recipe_search._search_page(candidate_query) + if not hits: + continue + preview_titles = [str(hit.get("title") or "") for hit in hits[:4] if str(hit.get("title") or "").strip()] + result_count = len(hits) + break + updates = { + "last_checked_at": store.now(), + "result_count": result_count, + "sample_titles": preview_titles, + } + if result_count > 0: + updates["status"] = "resolved-with-recipes" + updates["resolved_at"] = store.now() + saved = repo.update_failed_search(str(item.get("id")), updates) or {**item, **updates} + refreshed.append(saved) + return {"status": "ok", "items": refreshed} diff --git a/home/doris-kitchen/app/routes/api_search.py b/home/doris-kitchen/app/routes/api_search.py index 2346f4d..987b548 100644 --- a/home/doris-kitchen/app/routes/api_search.py +++ b/home/doris-kitchen/app/routes/api_search.py @@ -1,9 +1,38 @@ from fastapi import APIRouter, HTTPException, Request +from app.services.recipe_importer import normalize_recipe_from_url +from app.services.repository import repo +from app.services.scoring import extract_ingredients, extract_tags, score_recipe +from app.services.store import JsonStore from app.services.recipe_search import recipe_search router = APIRouter(prefix="/api/search", tags=["search"]) +store = JsonStore() +MIN_HEALTHY_RESULTS = 5 + + +def _build_suggestion(*, query: str, url: str, source_title: str, source_snippet: str, payload: dict) -> dict: + profile = repo.get_profile() + score, reasons = score_recipe(query, payload, profile) + return { + "id": store.new_id("suggestion"), + "title": payload.get("name") or source_title or url, + "url": url, + "source_query": query, + "source_title": source_title or url, + "source_snippet": source_snippet, + "payload": payload, + "ingredients": extract_ingredients(payload), + "tags": extract_tags(payload), + "score": score, + "score_reasons": reasons, + "status": "suggested", + "created_at": store.now(), + "updated_at": store.now(), + "kitchenowl_status": "not-imported", + "ready_for_auto_import": bool(profile.get("auto_import_enabled") and score >= int(profile.get("auto_import_threshold", 96))), + } @router.post("") @@ -13,12 +42,60 @@ async def run_search(request: Request): if not query: raise HTTPException(status_code=400, detail="Search query is required.") limit = int(payload.get("limit") or 6) + candidate_queries = recipe_search._candidate_queries(query) results = recipe_search.search(query, limit=limit) - return {"status": "ok", "query": query, "results": results} + failed_search = None + if len(results) < MIN_HEALTHY_RESULTS: + attention_type = "no-results" if not results else "low-result-count" + notes = "No recipe suggestions returned from the configured search lane." if not results else f"Search returned only {len(results)} unique recipes; more coverage needed." + existing = next( + ( + item for item in repo.list_failed_searches() + if str(item.get("query") or "").strip().lower() == query.lower() and str(item.get("status") or "open").startswith("open") + ), + None, + ) + failed_search = { + "id": str((existing or {}).get("id") or store.new_id("failed-search")), + "query": query, + "candidate_queries": candidate_queries, + "created_at": str((existing or {}).get("created_at") or store.now()), + "updated_at": store.now(), + "status": "open", + "attention_type": attention_type, + "result_count": len(results), + "sample_titles": [str(item.get("title") or "") for item in results[:4]], + "notes": notes, + } + if existing: + repo.update_failed_search(str(existing.get("id")), failed_search) + else: + repo.log_failed_search(failed_search) + return {"status": "ok", "query": query, "results": results, "failed_search": failed_search, "needs_attention": bool(failed_search)} + + +@router.post("/url") +async def ingest_url(request: Request): + payload = await request.json() + url = str(payload.get("url") or "").strip() + if not url.startswith("http://") and not url.startswith("https://"): + raise HTTPException(status_code=400, detail="Recipe URL must start with http:// or https://") + try: + normalized = normalize_recipe_from_url(url, timeout=20) + except Exception as exc: + raise HTTPException(status_code=400, detail=f"Could not normalize recipe link: {exc}") from exc + suggestion = _build_suggestion( + query="direct recipe link", + url=url, + source_title=url, + source_snippet="Recipe URL pasted directly into Doris Kitchen.", + payload=normalized, + ) + saved = repo.upsert_suggestion(suggestion) + return {"status": "ok", "url": url, "suggestion": saved} @router.post("/seed") async def seed_queue(): results = recipe_search.seed_queries() return {"status": "ok", "seeded": results, "count": len(results)} - diff --git a/home/doris-kitchen/app/routes/api_suggestions.py b/home/doris-kitchen/app/routes/api_suggestions.py index e822424..d517032 100644 --- a/home/doris-kitchen/app/routes/api_suggestions.py +++ b/home/doris-kitchen/app/routes/api_suggestions.py @@ -49,7 +49,8 @@ async def decide(suggestion_id: str, request: Request): if decision == "approve": profile = _apply_learning(profile, current, approved=True) import_now = bool(payload.get("import_to_kitchenowl", True)) - if import_now and kitchenowl.configured(): + already_linked = str(current.get("status") or "") == "approved" and str(current.get("kitchenowl_status") or "") in {"created", "duplicate", "updated"} + if import_now and kitchenowl.configured() and not already_linked: kitchenowl_result = kitchenowl.create_recipe(current.get("payload") or {}) current_status = "approved" elif decision == "reject": @@ -69,4 +70,3 @@ async def decide(suggestion_id: str, request: Request): }, ) return {"status": "ok", "suggestion": updated, "profile": profile, "kitchenowl": kitchenowl_result} - diff --git a/home/doris-kitchen/app/routes/ui.py b/home/doris-kitchen/app/routes/ui.py index e1014d1..ad1d039 100644 --- a/home/doris-kitchen/app/routes/ui.py +++ b/home/doris-kitchen/app/routes/ui.py @@ -23,3 +23,7 @@ async def search(request: Request): async def planner_page(request: Request): return templates.TemplateResponse("planner.html", {"request": request, "title": "Meal Planner"}) + +@router.get("/failures", response_class=HTMLResponse) +async def failures_page(request: Request): + return templates.TemplateResponse("failures.html", {"request": request, "title": "Failed Searches"}) diff --git a/home/doris-kitchen/app/services/kitchenowl.py b/home/doris-kitchen/app/services/kitchenowl.py index 139f332..da3736a 100644 --- a/home/doris-kitchen/app/services/kitchenowl.py +++ b/home/doris-kitchen/app/services/kitchenowl.py @@ -1,6 +1,9 @@ from __future__ import annotations import json +import mimetypes +import subprocess +import uuid from typing import Any from urllib import error, parse, request @@ -44,6 +47,69 @@ class KitchenOwlService: parsed = raw return exc.code, parsed + def _fetch_binary(self, url: str, timeout: int = 45) -> tuple[bytes, str]: + req = request.Request( + url, + headers={ + "User-Agent": "Mozilla/5.0 (Doris Kitchen)", + "Accept": "image/*,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + }, + ) + try: + with request.urlopen(req, timeout=timeout) as resp: + mime_type = resp.headers.get_content_type() or "application/octet-stream" + return resp.read(), mime_type + except error.HTTPError as exc: + if exc.code not in {403, 406, 429}: + raise + curl = subprocess.run( + [ + "curl", "-fsSL", "--max-time", str(timeout), + "-A", "Mozilla/5.0 (Doris Kitchen)", + "-H", "Accept: image/*,*/*;q=0.8", + "-H", "Accept-Language: en-US,en;q=0.9", + url, + ], + capture_output=True, + check=False, + ) + if curl.returncode != 0: + raise RuntimeError(f"Failed to fetch image URL: {url}") + guessed = mimetypes.guess_type(url)[0] or "application/octet-stream" + return curl.stdout, guessed + + def _upload_photo_from_url(self, photo_url: str) -> str | None: + photo_url = str(photo_url or "").strip() + if not photo_url.startswith(("http://", "https://")): + return photo_url or None + content, mime_type = self._fetch_binary(photo_url) + ext = mimetypes.guess_extension(mime_type) or ".jpg" + filename = f"doris-{uuid.uuid4().hex}{ext}" + boundary = f"----DorisBoundary{uuid.uuid4().hex}" + parts = [ + f"--{boundary}\r\n".encode("utf-8"), + f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'.encode("utf-8"), + f"Content-Type: {mime_type}\r\n\r\n".encode("utf-8"), + content, + f"\r\n--{boundary}--\r\n".encode("utf-8"), + ] + data = b"".join(parts) + req = request.Request( + f"{self.base_url}/api/upload", + data=data, + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/json", + "Content-Type": f"multipart/form-data; boundary={boundary}", + "User-Agent": "doris-kitchen/1.0", + }, + method="POST", + ) + with request.urlopen(req, timeout=60) as resp: + payload = json.loads(resp.read().decode("utf-8", "replace")) + return str(payload.get("filename") or "").strip() or None + def list_recipes(self) -> list[dict[str, Any]]: if not self.configured(): return [] @@ -64,15 +130,61 @@ class KitchenOwlService: return recipe return None + def _recipe_update_payload(self, existing: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: + uploaded_photo = self._upload_photo_from_url(payload.get("photo")) if payload.get("photo") else existing.get("photo") + return { + "name": payload.get("name") or existing.get("name"), + "description": payload.get("description") or existing.get("description") or "", + "source": payload.get("source") or existing.get("source"), + "visibility": int(payload.get("visibility", existing.get("visibility", 0) or 0)), + "items": payload.get("items") or [ + { + "name": item.get("name"), + "description": item.get("description", ""), + "optional": bool(item.get("optional", False)), + } + for item in (existing.get("items") or []) + ], + "tags": payload.get("tags") or [ + tag.get("name") if isinstance(tag, dict) else str(tag) + for tag in (existing.get("tags") or []) + ], + "cook_time": payload.get("cook_time", existing.get("cook_time")), + "prep_time": payload.get("prep_time", existing.get("prep_time")), + "time": payload.get("time", existing.get("time")), + "yields": payload.get("yields", existing.get("yields")), + "photo": uploaded_photo or existing.get("photo"), + "planned": bool(existing.get("planned", False)), + "planned_cooking_dates": existing.get("planned_cooking_dates") or [], + "planned_days": existing.get("planned_days") or [], + } + + def update_recipe(self, recipe_id: int | str, payload: dict[str, Any], existing: dict[str, Any] | None = None) -> dict[str, Any]: + if not self.configured(): + raise RuntimeError("KitchenOwl credentials are not configured.") + if existing is None: + status, existing = self._request("GET", f"/api/recipe/{recipe_id}") + if status != 200 or not isinstance(existing, dict): + return {"status": "error", "http_status": status, "recipe": existing} + body = self._recipe_update_payload(existing, payload) + status, updated = self._request("POST", f"/api/recipe/{recipe_id}", body=body, timeout=45) + return {"status": "updated" if status == 200 else "error", "http_status": status, "recipe": updated} + def create_recipe(self, payload: dict[str, Any]) -> dict[str, Any]: if not self.configured(): raise RuntimeError("KitchenOwl credentials are not configured.") existing = self.find_existing(payload) if existing: + if payload.get("photo") and not existing.get("photo"): + refreshed = self.update_recipe(existing.get("id"), payload, existing=existing) + if refreshed.get("status") == "updated": + return {"status": "updated", "recipe": refreshed.get("recipe")} return {"status": "duplicate", "recipe": existing} - status, created = self._request("POST", f"/api/household/{self.household_id}/recipe", body=payload, timeout=45) + create_payload = dict(payload) + if create_payload.get("photo"): + create_payload["photo"] = self._upload_photo_from_url(create_payload["photo"]) + status, created = self._request("POST", f"/api/household/{self.household_id}/recipe", body=create_payload, timeout=45) return {"status": "created" if status == 200 else "error", "http_status": status, "recipe": created} kitchenowl = KitchenOwlService() - diff --git a/home/doris-kitchen/app/services/recipe_importer.py b/home/doris-kitchen/app/services/recipe_importer.py index e8e2cb5..0bed5e7 100644 --- a/home/doris-kitchen/app/services/recipe_importer.py +++ b/home/doris-kitchen/app/services/recipe_importer.py @@ -8,6 +8,8 @@ from html.parser import HTMLParser from typing import Any from urllib import error, request +from app.services.tagging import curate_recipe_tags + UNIT_WORDS = { "teaspoon", "teaspoons", "tsp", "tsp.", "tablespoon", "tablespoons", "tbsp", "tbsp.", "cup", "cups", @@ -147,10 +149,110 @@ def coerce_text(value: Any) -> str: return unescape(str(value)).strip() -def slugify_tag(value: str) -> str: - value = value.strip().lower() - value = re.sub(r"[^a-z0-9]+", "-", value) - return value.strip("-")[:48] +def normalize_whitespace(value: str) -> str: + cleaned = re.sub(r"\s+", " ", value or "").strip() + return re.sub(r"(?<=[.!?])(?=[A-Z])", " ", cleaned) + + +def split_paragraphs(value: str) -> list[str]: + parts: list[str] = [] + for chunk in re.split(r"\n\s*\n|\r\n\r\n", value or ""): + cleaned = normalize_whitespace(chunk) + if cleaned: + parts.append(cleaned) + return parts + + +def extract_instruction_steps(value: Any) -> list[str]: + steps: list[str] = [] + + def visit(node: Any) -> None: + if node is None: + return + if isinstance(node, str): + for chunk in re.split(r"\n+", node): + cleaned = normalize_whitespace(chunk) + if cleaned: + steps.append(cleaned) + return + if isinstance(node, list): + for item in node: + visit(item) + return + if isinstance(node, dict): + if node.get("@type") == "HowToSection": + name = normalize_whitespace(coerce_text(node.get("name"))) + if name: + steps.append(f"{name}:") + visit(node.get("itemListElement")) + return + if node.get("@type") in {"HowToStep", "ListItem"}: + text = normalize_whitespace(coerce_text(node.get("text") or node.get("name") or node.get("item"))) + if text: + steps.append(text) + elif node.get("itemListElement") is not None: + visit(node.get("itemListElement")) + return + if node.get("itemListElement") is not None: + visit(node.get("itemListElement")) + return + text = normalize_whitespace(coerce_text(node.get("text") or node.get("name"))) + if text: + steps.append(text) + + visit(value) + deduped: list[str] = [] + seen: set[str] = set() + for step in steps: + key = step.lower() + if key in seen: + continue + seen.add(key) + deduped.append(step) + return deduped + + +def build_description(recipe_node: dict[str, Any]) -> str: + intro_parts = split_paragraphs(coerce_text(recipe_node.get("description"))) + intro = intro_parts[0] if intro_parts else "" + notes = intro_parts[1:] if len(intro_parts) > 1 else [] + steps = extract_instruction_steps(recipe_node.get("recipeInstructions")) + parts: list[str] = [] + if intro: + parts.append(intro) + if steps: + rendered_steps = [] + step_number = 1 + for idx, step in enumerate(steps, start=1): + if step.endswith(":"): + rendered_steps.append(step) + else: + rendered_steps.append(f"{step_number}. {step}") + step_number += 1 + parts.append("\n".join(rendered_steps)) + elif intro_parts[1:]: + parts.append("\n".join(intro_parts[1:])) + if notes: + parts.append("Notes:\n- " + "\n- ".join(notes)) + return "\n\n".join(part for part in parts if part).strip() + + +def extract_image_url(value: Any) -> str | None: + if isinstance(value, str): + cleaned = value.strip() + return cleaned or None + if isinstance(value, list): + for item in value: + found = extract_image_url(item) + if found: + return found + return None + if isinstance(value, dict): + for key in ("url", "contentUrl"): + found = extract_image_url(value.get(key)) + if found: + return found + return None def normalize_keywords(value: Any) -> list[str]: @@ -212,9 +314,7 @@ def parse_duration_minutes(value: Any) -> int | None: def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str | None = None) -> dict[str, Any]: name = coerce_text(recipe_node.get("name")) or (title_fallback or url) - desc = coerce_text(recipe_node.get("description")) - instructions = coerce_text(recipe_node.get("recipeInstructions")) - description = "\n\n".join(part for part in [desc, instructions] if part).strip() + description = build_description(recipe_node) items: list[dict[str, Any]] = [] seen_items: set[tuple[str, str]] = set() for raw in recipe_node.get("recipeIngredient") or []: @@ -232,13 +332,13 @@ def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str tags: list[str] = [] for key in TAG_KEYS: tags.extend(normalize_keywords(recipe_node.get(key))) - deduped_tags: list[str] = [] - seen_tags: set[str] = set() - for tag in tags: - slug = slugify_tag(tag) - if slug and slug not in seen_tags: - seen_tags.add(slug) - deduped_tags.append(slug) + deduped_tags = curate_recipe_tags( + name=name, + description=description, + items=items, + source=url, + existing_tags=tags, + ) payload = { "name": name[:128], "description": description[:20000], @@ -247,6 +347,9 @@ def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str "items": items, "tags": deduped_tags[:20], } + image_url = extract_image_url(recipe_node.get("image")) + if image_url: + payload["photo"] = image_url cook_time = parse_duration_minutes(recipe_node.get("cookTime")) prep_time = parse_duration_minutes(recipe_node.get("prepTime")) total_time = parse_duration_minutes(recipe_node.get("totalTime")) @@ -274,4 +377,3 @@ def normalize_recipe_from_url(url: str, timeout: int = 30) -> dict[str, Any]: if not recipe_node: raise RuntimeError(f"No schema.org Recipe JSON-LD found for {url}") return normalize_recipe(url, recipe_node, title) - diff --git a/home/doris-kitchen/app/services/recipe_search.py b/home/doris-kitchen/app/services/recipe_search.py index a5a6716..802668b 100644 --- a/home/doris-kitchen/app/services/recipe_search.py +++ b/home/doris-kitchen/app/services/recipe_search.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import xml.etree.ElementTree as ET from typing import Any from urllib import parse, request @@ -21,6 +22,9 @@ ALLOWED_DOMAINS = ( "spendwithpennies.com", "delish.com", "food.com", + "feelgoodfoodie.net", + "simplyrecipes.com", + "themediterraneandish.com", ) FALLBACK_SEED_URLS = [ "https://www.budgetbytes.com/curry-beef-with-peas/", @@ -28,13 +32,125 @@ FALLBACK_SEED_URLS = [ "https://www.spendwithpennies.com/ground-beef-stroganoff/", "https://www.budgetbytes.com/chicken-and-rice-skillet/", ] +QUERY_URL_FALLBACKS = { + "colcannon recipe": [ + "https://www.allrecipes.com/recipe/58642/dianes-colcannon/", + "https://www.spendwithpennies.com/colcannon-recipe-cabbage-and-potatoes/", + ], + "boxty recipe": [ + "https://www.delish.com/cooking/recipe-ideas/a42452906/boxty-recipe/", + ], + "coddle recipe": [ + "https://www.foodnetwork.com/recipes/food-network-kitchen/irish-coddle-12348168", + ], + "corned beef and cabbage recipe": [ + "https://www.spendwithpennies.com/corned-beef-and-cabbage/", + "https://www.thepioneerwoman.com/food-cooking/recipes/a11462/corned-beef-and-cabbage/", + ], + "shepherds pie recipe": [ + "https://natashaskitchen.com/shepherds-pie-recipe/", + ], + "kofta recipe ground beef": [ + "https://www.allrecipes.com/recipe/106030/kofta-kebabs/", + ], + "kafta recipe ground beef": [ + "https://feelgoodfoodie.net/recipe/beef-kafta/", + ], + "arayes recipe": [ + "https://www.recipetineats.com/arayes-lebanese-meat-stuffed-pita/", + "https://www.allrecipes.com/arayes-lebanese-crispy-meat-stuffed-pita-recipe-8635793", + ], + "hashweh recipe": [ + "https://www.themediterraneandish.com/lebanese-rice-hashweh-recipe/", + ], + "beef shawarma recipe": [ + "https://www.themediterraneandish.com/grilled-beef-shawarma/", + ], + "kofta kebabs recipe": [ + "https://www.allrecipes.com/recipe/106030/kofta-kebabs/", + ], + "kibbeh recipe": [ + "https://feelgoodfoodie.net/recipe/baked-kibbeh/", + ], +} +QUERY_STOPWORDS = { + "a", "an", "and", "for", "from", "in", "inspired", "meal", "meals", + "recipe", "recipes", "style", "using", "with", +} +CUE_WORDS = { + "asian", "irish", "italian", "mexican", "thai", "indian", "korean", + "japanese", "greek", "mediterranean", "southern", "cajun", "middle", "eastern", +} +TITLE_STOPWORDS = { + "a", "an", "and", "best", "classic", "easy", "for", "how", "make", + "recipe", "recipes", "style", "the", "to", "traditional", "slow", "cooker", "s", +} class RecipeSearchService: def __init__(self) -> None: self.store = JsonStore() + def _dish_key(self, title: str, payload: dict[str, Any]) -> str: + base = str(title or payload.get("name") or "").lower() + base = re.sub(r"\([^)]*\)", " ", base) + base = re.sub(r"[^a-z0-9\s-]", " ", base) + tokens = [token for token in re.split(r"\s+", base) if token and token not in TITLE_STOPWORDS] + if len(tokens) > 1 and tokens[0].endswith("s"): + tokens = tokens[1:] + canonical_tokens = sorted(dict.fromkeys(tokens)) + return " ".join(canonical_tokens[:8]).strip() + + def _candidate_queries(self, query: str) -> list[str]: + normalized = re.sub(r"[^a-z0-9\s-]", " ", query.lower()) + normalized = re.sub(r"\s+", " ", normalized).strip() + tokens = [token for token in normalized.split() if token] + filtered = [token for token in tokens if token not in QUERY_STOPWORDS] + cues = [token for token in filtered if token in CUE_WORDS] + ingredients = [token for token in filtered if token not in CUE_WORDS] + expansion_queries: list[str] = [] + filtered_set = set(filtered) + if {"irish", "cabbage", "potatoes"} <= filtered_set: + expansion_queries.extend([ + "colcannon recipe", + "boxty recipe", + "coddle recipe", + "corned beef and cabbage recipe", + "shepherds pie recipe", + ]) + if {"middle", "eastern", "ground", "beef"} <= filtered_set: + expansion_queries.extend([ + "kofta recipe ground beef", + "kafta recipe ground beef", + "middle eastern beef kofta", + "arayes recipe", + "hashweh recipe", + "beef shawarma recipe", + "kofta kebabs recipe", + "kibbeh recipe", + ]) + variants = [ + query.strip(), + " ".join(filtered), + " ".join(cues + ingredients + ["recipe"]).strip(), + " ".join(ingredients + ["recipe"]).strip(), + ] + expansion_queries + deduped: list[str] = [] + seen: set[str] = set() + for variant in variants: + cleaned = variant.strip() + if not cleaned: + continue + key = cleaned.lower() + if key in seen: + continue + seen.add(key) + deduped.append(cleaned) + return deduped + def _search_page(self, query: str) -> list[dict[str, str]]: + if query in QUERY_URL_FALLBACKS: + return [{"url": url, "title": url, "snippet": ""} for url in QUERY_URL_FALLBACKS[query]] req = request.Request( SEARCH_URL + parse.quote(query + ENGLISH_HINT), headers={ @@ -69,34 +185,48 @@ class RecipeSearchService: limit = limit or settings.search_result_limit profile = repo.get_profile() results: list[dict[str, Any]] = [] - for hit in self._search_page(query): + seen_urls: set[str] = set() + seen_dishes: set[str] = set() + for candidate_query in self._candidate_queries(query): + before_count = len(results) + for hit in self._search_page(candidate_query): + if len(results) >= limit: + break + if hit["url"] in seen_urls: + continue + try: + payload = normalize_recipe_from_url(hit["url"], timeout=12) + except Exception: + continue + dish_key = self._dish_key(str(payload.get("name") or hit["title"] or ""), payload) + if dish_key and dish_key in seen_dishes: + continue + score, reasons = score_recipe(query, payload, profile) + suggestion = { + "id": self.store.new_id("suggestion"), + "title": payload.get("name") or hit["title"], + "url": hit["url"], + "source_query": query, + "source_title": hit["title"], + "source_snippet": hit["snippet"], + "payload": payload, + "ingredients": extract_ingredients(payload), + "tags": extract_tags(payload), + "score": score, + "score_reasons": reasons, + "status": "suggested", + "created_at": self.store.now(), + "updated_at": self.store.now(), + "kitchenowl_status": "not-imported", + "ready_for_auto_import": bool(profile.get("auto_import_enabled") and score >= int(profile.get("auto_import_threshold", settings.auto_import_threshold))), + } + saved = repo.upsert_suggestion(suggestion) + results.append(saved) + seen_urls.add(hit["url"]) + if dish_key: + seen_dishes.add(dish_key) if len(results) >= limit: break - try: - payload = normalize_recipe_from_url(hit["url"], timeout=12) - except Exception: - continue - score, reasons = score_recipe(query, payload, profile) - suggestion = { - "id": self.store.new_id("suggestion"), - "title": payload.get("name") or hit["title"], - "url": hit["url"], - "source_query": query, - "source_title": hit["title"], - "source_snippet": hit["snippet"], - "payload": payload, - "ingredients": extract_ingredients(payload), - "tags": extract_tags(payload), - "score": score, - "score_reasons": reasons, - "status": "suggested", - "created_at": self.store.now(), - "updated_at": self.store.now(), - "kitchenowl_status": "not-imported", - "ready_for_auto_import": bool(profile.get("auto_import_enabled") and score >= int(profile.get("auto_import_threshold", settings.auto_import_threshold))), - } - repo.upsert_suggestion(suggestion) - results.append(suggestion) return results def _fallback_seed(self) -> list[dict[str, Any]]: @@ -126,8 +256,8 @@ class RecipeSearchService: "kitchenowl_status": "not-imported", "ready_for_auto_import": False, } - repo.upsert_suggestion(suggestion) - results.append(suggestion) + saved = repo.upsert_suggestion(suggestion) + results.append(saved) return results def seed_queries(self) -> list[dict[str, Any]]: diff --git a/home/doris-kitchen/app/services/repository.py b/home/doris-kitchen/app/services/repository.py index 2880148..dfe58f2 100644 --- a/home/doris-kitchen/app/services/repository.py +++ b/home/doris-kitchen/app/services/repository.py @@ -58,7 +58,19 @@ class JsonRepository: current = self.list_suggestions() for idx, existing in enumerate(current): if existing.get("url") == item.get("url"): - merged = {**existing, **item} + merged = {**item} + for key in ( + "status", + "decision_at", + "kitchenowl_status", + "kitchenowl_result", + ): + if existing.get(key) is not None: + merged[key] = existing.get(key) + if existing.get("id"): + merged["id"] = existing["id"] + if existing.get("created_at"): + merged["created_at"] = existing["created_at"] current[idx] = merged self.store.save("suggestions", current) return merged @@ -83,5 +95,25 @@ class JsonRepository: items = sorted(self.list_suggestions(), key=lambda item: item.get("updated_at") or item.get("created_at") or "", reverse=True) return items[:limit] + def list_failed_searches(self) -> list[dict[str, Any]]: + return self.store.load("failed_searches", []) + + def log_failed_search(self, item: dict[str, Any]) -> dict[str, Any]: + current = self.list_failed_searches() + current.append(item) + current = sorted(current, key=lambda entry: entry.get("created_at") or "", reverse=True)[:50] + self.store.save("failed_searches", current) + return item + + def recent_failed_searches(self, limit: int = 12) -> list[dict[str, Any]]: + return self.list_failed_searches()[:limit] + + def update_failed_search(self, failed_search_id: str, updates: dict[str, Any]) -> dict[str, Any] | None: + return self.store.update_collection_item( + "failed_searches", + lambda item: str(item.get("id")) == str(failed_search_id), + lambda item: {**item, **updates}, + ) + repo = JsonRepository() diff --git a/home/doris-kitchen/app/services/tagging.py b/home/doris-kitchen/app/services/tagging.py new file mode 100644 index 0000000..97b7d1c --- /dev/null +++ b/home/doris-kitchen/app/services/tagging.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import re +from typing import Any + + +def slugify_tag(value: str) -> str: + value = value.strip().lower() + value = re.sub(r"[^a-z0-9]+", "-", value) + return value.strip("-")[:48] + + +CUISINE_TAGS = { + "asian": {"asian"}, + "chinese": {"asian", "chinese"}, + "indian": {"asian", "indian"}, + "irish": {"irish"}, + "italian": {"italian"}, + "japanese": {"asian", "japanese"}, + "korean": {"asian", "korean"}, + "lebanese": {"middle-eastern", "lebanese"}, + "mediterranean": {"mediterranean"}, + "mexican": {"mexican"}, + "middle eastern": {"middle-eastern"}, + "middle-eastern": {"middle-eastern"}, + "thai": {"asian", "thai"}, +} + +PROTEIN_TAGS = { + "beef": "beef", + "chicken": "chicken", + "lamb": "lamb", + "pork": "pork", + "shrimp": "seafood", + "tofu": "vegetarian", + "turkey": "turkey", +} + +COURSE_HINTS = { + "appetizer": "appetizer", + "breakfast": "breakfast", + "dessert": "dessert", + "main course": "main-course", + "main dish": "main-course", + "salad": "salad", + "side dish": "side-dish", + "soup": "soup", +} + +METHOD_HINTS = { + "baked": "baked", + "grilled": "grilled", + "noodle": "noodles", + "ramen": "noodles", + "rice": "rice", + "skillet": "skillet", + "slow cooker": "slow-cooker", + "stir fry": "stir-fry", + "stir-fry": "stir-fry", +} + +SWEET_HINTS = ( + "brown sugar", "cake", "caramel", "chocolate", "cookie", "dessert", "frosting", + "honey", "icing", "maple syrup", "molasses", "muffin", "pie", "pudding", "sweet", + "vanilla", "whipped cream", +) + +SAVORY_HINTS = ( + "beef", "broth", "butter", "cheese", "chicken", "cilantro", "cumin", "garlic", "ginger", + "ground beef", "lamb", "meat", "noodle", "onion", "paprika", "parsley", "pine nuts", + "rice", "salt", "savory", "soy sauce", "stock", "thyme", +) + + +def _flatten_texts(items: list[dict[str, Any]] | None) -> str: + flattened: list[str] = [] + for item in items or []: + flattened.append(str(item.get("name") or "")) + flattened.append(str(item.get("description") or "")) + return " ".join(flattened) + + +def curate_recipe_tags( + *, + name: str, + description: str, + items: list[dict[str, Any]] | None, + source: str = "", + existing_tags: list[str] | None = None, +) -> list[str]: + haystack = " ".join( + part for part in [ + str(name or ""), + str(description or ""), + _flatten_texts(items), + " ".join(existing_tags or []), + str(source or ""), + ] + if part + ).lower() + + curated: list[str] = [] + seen: set[str] = set() + + def add(tag: str) -> None: + slug = slugify_tag(tag) + if slug and slug not in seen: + seen.add(slug) + curated.append(slug) + + for tag in existing_tags or []: + add(tag) + + for key, expanded in CUISINE_TAGS.items(): + if key in haystack: + for tag in expanded: + add(tag) + + for key, tag in COURSE_HINTS.items(): + if key in haystack: + add(tag) + + for key, tag in METHOD_HINTS.items(): + if key in haystack: + add(tag) + + for key, tag in PROTEIN_TAGS.items(): + if key in haystack: + add(tag) + + if "kibbeh" in haystack: + add("middle-eastern") + add("lebanese") + add("main-course") + + if "larb" in haystack or "laab" in haystack: + add("thai") + add("main-course") + + if "curry" in haystack: + add("curry") + add("main-course") + + if "pasta" in haystack: + add("pasta") + + sweet_score = sum(1 for hint in SWEET_HINTS if hint in haystack) + savory_score = sum(1 for hint in SAVORY_HINTS if hint in haystack) + if sweet_score > savory_score and sweet_score > 0: + add("sweet") + elif savory_score > 0: + add("savory") + + if "dessert" not in seen and "sweet" not in seen and "main-course" not in seen and "side-dish" not in seen: + add("main-course") + + return curated[:20] diff --git a/home/doris-kitchen/app/static/app.css b/home/doris-kitchen/app/static/app.css index 546fd31..29ceaec 100644 --- a/home/doris-kitchen/app/static/app.css +++ b/home/doris-kitchen/app/static/app.css @@ -133,6 +133,20 @@ nav a, gap: 14px; } +.search-examples { + margin-top: 16px; + display: grid; + gap: 10px; +} + +.search-examples .chip-list { + align-items: flex-start; +} + +.search-examples button { + text-align: left; +} + label { display: grid; gap: 8px; @@ -219,7 +233,8 @@ button.danger { } .suggestion-grid, -.plan-grid { +.plan-grid, +.failure-list { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 18px; @@ -227,13 +242,35 @@ button.danger { .suggestion-card, .plan-card, -.plan-summary { +.plan-summary, +.failure-card { border: 1px solid var(--line); border-radius: 18px; background: rgba(255, 255, 255, 0.55); padding: 18px; } +.failure-results { + margin-top: 14px; + padding-top: 14px; + border-top: 1px solid var(--line); +} + +.failure-results .suggestion-grid, +.failure-results.suggestion-grid { + margin-top: 8px; +} + +.status-open { + background: rgba(167, 71, 53, 0.12); + color: var(--rose); +} + +.status-resolved { + background: rgba(108, 122, 61, 0.14); + color: var(--olive); +} + .suggestion-card h3, .plan-card h3 { margin-bottom: 8px; @@ -260,4 +297,3 @@ button.danger { grid-template-columns: repeat(2, minmax(0, 1fr)); } } - diff --git a/home/doris-kitchen/app/static/app.js b/home/doris-kitchen/app/static/app.js index 7fad802..4a7da1c 100644 --- a/home/doris-kitchen/app/static/app.js +++ b/home/doris-kitchen/app/static/app.js @@ -51,6 +51,41 @@ function renderSuggestionCards(target, items) { }).join(''); } +function renderFailureCards(target, items) { + if (!target) return; + if (!items.length) { + target.textContent = 'No failed searches logged.'; + target.classList.add('muted'); + return; + } + target.classList.remove('muted'); + target.innerHTML = items.map((item) => { + const status = String(item.status || 'open'); + const resolved = status === 'resolved-with-recipes'; + const titles = (item.sample_titles || []).map((entry) => '
Logged query
+${escapeHtml(status)}
+Logged: ${escapeHtml(item.created_at || '')}
+Current matching recipes: ${escapeHtml(item.result_count || 0)}
+No actionable recipes left in this review set.
'; + failurePanel.classList.remove('muted'); + } + const failureCard = failurePanel.closest('[data-failure-id]'); + if (failureCard) { + const count = failureCard.querySelector('.failure-count'); + if (count) { + const currentText = count.textContent || ''; + const match = currentText.match(/(\d+)/); + if (match) { + const nextCount = Math.max(0, Number(match[1]) - 1); + count.textContent = 'Current matching recipes: ' + String(nextCount); + } + } + } + } if (document.getElementById('search-results')) { await loadSuggestionsInto(document.getElementById('search-results')); } + return data; } catch (error) { alert(error.message); } finally { @@ -117,12 +176,70 @@ async function loadSuggestionsInto(target) { renderSuggestionCards(target, data.items || []); } +async function loadFailuresInto(target) { + const data = await fetchJson('/api/failed-searches'); + renderFailureCards(target, data.items || []); +} + +async function loadFailureResults(failureId, target, button) { + if (!target || !failureId) return; + if (button) { + button.disabled = true; + button.textContent = 'Loading…'; + } + target.textContent = 'Loading recipes for this search…'; + target.classList.add('muted'); + try { + const data = await fetchJson('/api/failed-searches/' + encodeURIComponent(failureId) + '/results'); + const results = data.results || []; + if (!results.length) { + target.innerHTML = 'Still no usable recipes for this search.
'; + target.classList.remove('muted'); + } else { + renderSuggestionCards(target, results); + } + const card = target.closest('[data-failure-id]'); + if (card && data.item) { + const badge = card.querySelector('.pill'); + const count = card.querySelector('.failure-count'); + const list = card.querySelector('.failure-title-list'); + if (badge) { + const resolved = String(data.item.status || '') === 'resolved-with-recipes'; + badge.textContent = String(data.item.status || 'open'); + badge.className = 'pill ' + (resolved ? 'status-resolved' : 'status-open'); + } + if (count) { + count.textContent = 'Current matching recipes: ' + String(data.item.result_count || 0); + } + if (list) { + const titles = (data.item.sample_titles || []).map((entry) => '