Expand Doris Kitchen import, failures, and tag curation

home/doris-kitchen/app/services/recipe_importer.py

@@ -8,6 +8,8 @@ from html.parser import HTMLParser
 from typing import Any
 from urllib import error, request
 
+from app.services.tagging import curate_recipe_tags
+
 
 UNIT_WORDS = {
     "teaspoon", "teaspoons", "tsp", "tsp.", "tablespoon", "tablespoons", "tbsp", "tbsp.", "cup", "cups",
@@ -147,10 +149,110 @@ def coerce_text(value: Any) -> str:
     return unescape(str(value)).strip()
 
 
-def slugify_tag(value: str) -> str:
-    value = value.strip().lower()
-    value = re.sub(r"[^a-z0-9]+", "-", value)
-    return value.strip("-")[:48]
+def normalize_whitespace(value: str) -> str:
+    cleaned = re.sub(r"\s+", " ", value or "").strip()
+    return re.sub(r"(?<=[.!?])(?=[A-Z])", " ", cleaned)
+
+
+def split_paragraphs(value: str) -> list[str]:
+    parts: list[str] = []
+    for chunk in re.split(r"\n\s*\n|\r\n\r\n", value or ""):
+        cleaned = normalize_whitespace(chunk)
+        if cleaned:
+            parts.append(cleaned)
+    return parts
+
+
+def extract_instruction_steps(value: Any) -> list[str]:
+    steps: list[str] = []
+
+    def visit(node: Any) -> None:
+        if node is None:
+            return
+        if isinstance(node, str):
+            for chunk in re.split(r"\n+", node):
+                cleaned = normalize_whitespace(chunk)
+                if cleaned:
+                    steps.append(cleaned)
+            return
+        if isinstance(node, list):
+            for item in node:
+                visit(item)
+            return
+        if isinstance(node, dict):
+            if node.get("@type") == "HowToSection":
+                name = normalize_whitespace(coerce_text(node.get("name")))
+                if name:
+                    steps.append(f"{name}:")
+                visit(node.get("itemListElement"))
+                return
+            if node.get("@type") in {"HowToStep", "ListItem"}:
+                text = normalize_whitespace(coerce_text(node.get("text") or node.get("name") or node.get("item")))
+                if text:
+                    steps.append(text)
+                elif node.get("itemListElement") is not None:
+                    visit(node.get("itemListElement"))
+                return
+            if node.get("itemListElement") is not None:
+                visit(node.get("itemListElement"))
+                return
+            text = normalize_whitespace(coerce_text(node.get("text") or node.get("name")))
+            if text:
+                steps.append(text)
+
+    visit(value)
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for step in steps:
+        key = step.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(step)
+    return deduped
+
+
+def build_description(recipe_node: dict[str, Any]) -> str:
+    intro_parts = split_paragraphs(coerce_text(recipe_node.get("description")))
+    intro = intro_parts[0] if intro_parts else ""
+    notes = intro_parts[1:] if len(intro_parts) > 1 else []
+    steps = extract_instruction_steps(recipe_node.get("recipeInstructions"))
+    parts: list[str] = []
+    if intro:
+        parts.append(intro)
+    if steps:
+        rendered_steps = []
+        step_number = 1
+        for idx, step in enumerate(steps, start=1):
+            if step.endswith(":"):
+                rendered_steps.append(step)
+            else:
+                rendered_steps.append(f"{step_number}. {step}")
+                step_number += 1
+        parts.append("\n".join(rendered_steps))
+    elif intro_parts[1:]:
+        parts.append("\n".join(intro_parts[1:]))
+    if notes:
+        parts.append("Notes:\n- " + "\n- ".join(notes))
+    return "\n\n".join(part for part in parts if part).strip()
+
+
+def extract_image_url(value: Any) -> str | None:
+    if isinstance(value, str):
+        cleaned = value.strip()
+        return cleaned or None
+    if isinstance(value, list):
+        for item in value:
+            found = extract_image_url(item)
+            if found:
+                return found
+        return None
+    if isinstance(value, dict):
+        for key in ("url", "contentUrl"):
+            found = extract_image_url(value.get(key))
+            if found:
+                return found
+    return None
 
 
 def normalize_keywords(value: Any) -> list[str]:
@@ -212,9 +314,7 @@ def parse_duration_minutes(value: Any) -> int | None:
 
 def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str | None = None) -> dict[str, Any]:
     name = coerce_text(recipe_node.get("name")) or (title_fallback or url)
-    desc = coerce_text(recipe_node.get("description"))
-    instructions = coerce_text(recipe_node.get("recipeInstructions"))
-    description = "\n\n".join(part for part in [desc, instructions] if part).strip()
+    description = build_description(recipe_node)
     items: list[dict[str, Any]] = []
     seen_items: set[tuple[str, str]] = set()
     for raw in recipe_node.get("recipeIngredient") or []:
@@ -232,13 +332,13 @@ def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str
     tags: list[str] = []
     for key in TAG_KEYS:
         tags.extend(normalize_keywords(recipe_node.get(key)))
-    deduped_tags: list[str] = []
-    seen_tags: set[str] = set()
-    for tag in tags:
-        slug = slugify_tag(tag)
-        if slug and slug not in seen_tags:
-            seen_tags.add(slug)
-            deduped_tags.append(slug)
+    deduped_tags = curate_recipe_tags(
+        name=name,
+        description=description,
+        items=items,
+        source=url,
+        existing_tags=tags,
+    )
     payload = {
         "name": name[:128],
         "description": description[:20000],
@@ -247,6 +347,9 @@ def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str
         "items": items,
         "tags": deduped_tags[:20],
     }
+    image_url = extract_image_url(recipe_node.get("image"))
+    if image_url:
+        payload["photo"] = image_url
     cook_time = parse_duration_minutes(recipe_node.get("cookTime"))
     prep_time = parse_duration_minutes(recipe_node.get("prepTime"))
     total_time = parse_duration_minutes(recipe_node.get("totalTime"))
@@ -274,4 +377,3 @@ def normalize_recipe_from_url(url: str, timeout: int = 30) -> dict[str, Any]:
     if not recipe_node:
         raise RuntimeError(f"No schema.org Recipe JSON-LD found for {url}")
     return normalize_recipe(url, recipe_node, title)
-