From 458989e6cd3a704a5995ca22a5dff8218e3be0e0 Mon Sep 17 00:00:00 2001 From: Fizzlepoof Date: Fri, 15 May 2026 03:03:58 +0000 Subject: [PATCH] Add Doris KitchenOwl recipe importer --- automation/.env.example | 20 + automation/README.md | 88 ++++ automation/bin/kitchenowl_recipe_import.py | 475 ++++++++++++++++++++ docs/operations/KITCHENOWL_RECIPE_IMPORT.md | 137 ++++++ docs/planning/TODO.md | 32 +- 5 files changed, 737 insertions(+), 15 deletions(-) create mode 100644 automation/README.md create mode 100755 automation/bin/kitchenowl_recipe_import.py create mode 100644 docs/operations/KITCHENOWL_RECIPE_IMPORT.md diff --git a/automation/.env.example b/automation/.env.example index fbb9b2d..9cb5401 100644 --- a/automation/.env.example +++ b/automation/.env.example @@ -15,3 +15,23 @@ PAPERLESS_TAG_SUBMISSION_KIND_MAP_JSON={} PAPERLESS_DOCUMENT_TYPE_PAPER_KIND_MAP_JSON={} PAPERLESS_CORRESPONDENT_CLASS_MAP_JSON={} NODE_FUNCTION_ALLOW_BUILTIN=crypto + +# KitchenOwl recipe importer +KITCHENOWL_BASE_URL=https://owl.paccoco.com +KITCHENOWL_API_TOKEN=CHANGE_ME +KITCHENOWL_HOUSEHOLD_ID=1 + +# Backup automation scaffolding +SERENITY_BACKUP_HOST=root@10.5.1.5 +SERENITY_BACKUP_ROOT=/mnt/user/backups/plausible-deniability +PD_APPDATA_ROOT=/mnt/docker-ssd/docker/appdata +PD_DATABASES_ROOT=/mnt/docker-ssd/docker/databases +PD_TANK_DOCKER_ROOT=/mnt/tank/docker +PD_DB_DUMP_ROOT=/mnt/tank/docker/backups/db-dumps +POSTGRES_CONTAINER=shared-postgres +POSTGRES_DB_LIST="n8n paperless" +BACKUP_SSH_KEY=/home/truenas_admin/.ssh/serenity_backup_ed25519 +BACKUP_KNOWN_HOSTS=/home/truenas_admin/.ssh/known_hosts +DOCKER_BIN=/usr/bin/docker +SUDO_BIN=/usr/bin/sudo +USE_SUDO_FOR_DOCKER=true diff --git a/automation/README.md b/automation/README.md new file mode 100644 index 0000000..37899c1 --- /dev/null +++ b/automation/README.md @@ -0,0 +1,88 @@ +# Automation helpers + +This directory includes repo-side helpers for homelab operational tasks. + +## KitchenOwl recipe import helper + +- `bin/kitchenowl_recipe_import.py` — Doris-managed KitchenOwl recipe importer + - tries KitchenOwl's own scrape endpoint first + - falls back to direct page fetch + schema.org `Recipe` JSON-LD parsing + - normalizes ingredients/tags/timings into KitchenOwl's create-recipe payload + - supports dry-run by default and live create with `--create` + +See `docs/operations/KITCHENOWL_RECIPE_IMPORT.md` for setup and usage. + +## Backup scripts + +- `bin/pd_backup_postgres.sh` — creates gzipped `pg_dump` exports for the configured Postgres DB list (quote space-separated DB names in `.env`, e.g. `POSTGRES_DB_LIST="n8n paperless"`) +- `bin/pd_backup_sync_to_serenity.sh` — `rsync`s configured backup/config roots to Serenity +- `bin/run_pd_backups.sh` — wrapper that runs both steps in order + +These scripts are staged repo-side only. They still need: + +1. real values in `.env` +2. a writable dump destination on PD +3. SSH trust / key path for Serenity (`root@10.5.1.5` by default) + - recommended known_hosts path on PD: `/home/truenas_admin/.ssh/known_hosts` +4. a real scheduler (cron/systemd timer/n8n) on the live host + +## PD / TrueNAS deployment recommendation + +Use a plain cron job on PD, not n8n and not a custom systemd timer. + +Reasons: + +- survives n8n outages +- keeps the backup runner close to the compose/data host +- avoids extra appliance fights on TrueNAS SCALE +- matches the existing shell-first operational style on PD + +Preferred mode on PD: **root cron**. + +Why: the dump path under `/mnt/tank/...` and Docker access are typically cleaner from root on TrueNAS than from a limited operator account. + +TrueNAS-specific notes: + +- scripts use `/usr/bin/bash` explicitly +- default Docker path is `/usr/bin/docker` +- if run as root, scripts call Docker directly +- if run as a non-root user, default behavior is `sudo -n /usr/bin/docker ...` +- keep all live paths under `/mnt/...`; never rely on rootfs write locations + +## Example live flow + +```bash +cd /mnt/docker-ssd/docker/compose/automation +cp .env.example .env +chmod 600 .env +mkdir -p /mnt/tank/docker/backups/db-dumps +bin/run_pd_backups.sh +``` + +## Recommended PD cron block + +Install in **root's crontab on PD**: + +```cron +# BEGIN PD BACKUPS +15 2 * * * cd /mnt/docker-ssd/docker/compose/automation && /usr/bin/bash bin/run_pd_backups.sh >> /mnt/tank/docker/backups/pd-backups.log 2>&1 +# END PD BACKUPS +``` + +Recommended first-run checklist: + +```bash +cd /mnt/docker-ssd/docker/compose/automation +cp .env.example .env # if not already present +chmod 600 .env +mkdir -p /mnt/tank/docker/backups/db-dumps +/usr/bin/bash bin/run_pd_backups.sh +tail -100 /mnt/tank/docker/backups/pd-backups.log +``` + +## Safety notes + +- DB dumps are the priority restore artifacts; do not rely only on raw volume copies. +- The sync script uses `rsync --delete` inside the destination backup root, so point it at a dedicated backup path. +- Keep `.env` and SSH material out of git. +- If cron runs under a non-root PD account, `sudo -n /usr/bin/docker` must work or the DB dump step will fail. diff --git a/automation/bin/kitchenowl_recipe_import.py b/automation/bin/kitchenowl_recipe_import.py new file mode 100755 index 0000000..fc7b67a --- /dev/null +++ b/automation/bin/kitchenowl_recipe_import.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +"""Doris-managed KitchenOwl recipe importer. + +Flow: +1. Try KitchenOwl's native scrape endpoint for supported sites. +2. If that fails, fetch the page and extract schema.org Recipe JSON-LD. +3. Normalize ingredients/tags into KitchenOwl's create-recipe payload. +4. Optionally create the recipe via the KitchenOwl API. + +Secrets stay in env vars or a local .env, never in git. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import textwrap +from html import unescape +from html.parser import HTMLParser +from pathlib import Path +from typing import Any +from urllib import error, parse, request + +DEFAULT_TIMEOUT = 30 +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent.parent +DEFAULT_ENV_PATH = REPO_ROOT / 'automation' / '.env' + +UNIT_WORDS = { + 'teaspoon','teaspoons','tsp','tsp.','tablespoon','tablespoons','tbsp','tbsp.','cup','cups', + 'oz','ounce','ounces','lb','lbs','pound','pounds','gram','grams','g','kg','ml','l','pinch', + 'dash','clove','cloves','can','cans','package','packages','pkg','slice','slices','stalk','stalks' +} +STOPWORDS = {'fresh','large','small','medium','optional','to','taste','for','the','a','an'} +TAG_KEYS = ('recipeCategory', 'recipeCuisine', 'keywords') + + +class ScriptExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.in_script = False + self.script_type = None + self._buf: list[str] = [] + self.scripts: list[tuple[str | None, str]] = [] + self.title: str | None = None + self._in_title = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attrs_d = dict(attrs) + if tag == 'script': + self.in_script = True + self.script_type = attrs_d.get('type') + self._buf = [] + elif tag == 'title': + self._in_title = True + + def handle_endtag(self, tag: str) -> None: + if tag == 'script' and self.in_script: + self.scripts.append((self.script_type, ''.join(self._buf))) + self.in_script = False + self.script_type = None + self._buf = [] + elif tag == 'title': + self._in_title = False + + def handle_data(self, data: str) -> None: + if self.in_script: + self._buf.append(data) + elif self._in_title: + self.title = (self.title or '') + data + + +def load_env_file(path: Path) -> None: + if not path.exists(): + return + for raw_line in path.read_text(encoding='utf-8').splitlines(): + line = raw_line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + key, value = line.split('=', 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + os.environ.setdefault(key, value) + + +def env(name: str, default: str | None = None) -> str | None: + return os.environ.get(name, default) + + +def api_request(method: str, url: str, token: str, *, params: dict[str, Any] | None = None, + body: dict[str, Any] | None = None, timeout: int = DEFAULT_TIMEOUT) -> tuple[int, Any, dict[str, str]]: + if params: + url += ('&' if '?' in url else '?') + parse.urlencode(params, doseq=True) + data = None + headers = { + 'Authorization': f'Bearer {token}', + 'User-Agent': 'doris-kitchenowl-importer/1.0', + 'Accept': 'application/json', + } + if body is not None: + data = json.dumps(body).encode('utf-8') + headers['Content-Type'] = 'application/json' + req = request.Request(url, data=data, method=method.upper(), headers=headers) + try: + with request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode('utf-8', 'replace') + ctype = resp.headers.get('Content-Type', '') + payload = json.loads(raw) if 'json' in ctype else raw + return resp.status, payload, dict(resp.headers.items()) + except error.HTTPError as exc: + raw = exc.read().decode('utf-8', 'replace') + try: + payload = json.loads(raw) + except Exception: + payload = raw + return exc.code, payload, dict(exc.headers.items()) + + +def fetch_url(url: str, timeout: int = DEFAULT_TIMEOUT) -> str: + req = request.Request( + url, + headers={ + 'User-Agent': 'Mozilla/5.0 (Doris KitchenOwl Importer)', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + }, + ) + try: + with request.urlopen(req, timeout=timeout) as resp: + charset = resp.headers.get_content_charset() or 'utf-8' + return resp.read().decode(charset, 'replace') + except error.HTTPError as exc: + if exc.code not in {403, 406, 429}: + raise + curl = subprocess.run( + [ + 'curl', '-fsSL', '--max-time', str(timeout), + '-A', 'Mozilla/5.0 (Doris KitchenOwl Importer)', + '-H', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + '-H', 'Accept-Language: en-US,en;q=0.9', + url, + ], + check=False, + capture_output=True, + ) + if curl.returncode != 0: + raise RuntimeError(f'Failed to fetch URL via urllib/curl: {url} ({curl.stderr.decode("utf-8", "replace").strip()})') + return curl.stdout.decode('utf-8', 'replace') + + +def flatten_graph(node: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + if isinstance(node, list): + for item in node: + out.extend(flatten_graph(item)) + elif isinstance(node, dict): + if '@graph' in node: + out.extend(flatten_graph(node['@graph'])) + else: + out.append(node) + return out + + +def is_recipe_node(node: dict[str, Any]) -> bool: + t = node.get('@type') + if isinstance(t, list): + return 'Recipe' in t + return t == 'Recipe' + + +def choose_recipe_node(nodes: list[dict[str, Any]]) -> dict[str, Any] | None: + recipes = [n for n in nodes if is_recipe_node(n)] + if not recipes: + return None + recipes.sort(key=lambda r: len(r.get('recipeIngredient') or []), reverse=True) + return recipes[0] + + +def parse_json_ld_recipe(html: str) -> tuple[dict[str, Any] | None, str | None]: + parser = ScriptExtractor() + parser.feed(html) + nodes: list[dict[str, Any]] = [] + for script_type, content in parser.scripts: + if script_type and 'ld+json' not in script_type: + continue + content = content.strip() + if not content: + continue + try: + data = json.loads(content) + except Exception: + continue + nodes.extend(flatten_graph(data)) + return choose_recipe_node(nodes), (parser.title.strip() if parser.title else None) + + +DURATION_RE = re.compile(r'^P(?:T(?:(?P\d+)H)?(?:(?P\d+)M)?(?:(?P\d+)S)?)?$') + + +def parse_duration_minutes(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, (int, float)): + return int(value) + if not isinstance(value, str): + return None + value = value.strip() + if not value: + return None + m = DURATION_RE.match(value) + if m: + hours = int(m.group('hours') or 0) + minutes = int(m.group('minutes') or 0) + seconds = int(m.group('seconds') or 0) + total = hours * 60 + minutes + (1 if seconds >= 30 else 0) + return total or None + num = re.search(r'(\d+)', value) + return int(num.group(1)) if num else None + + +def coerce_text(value: Any) -> str: + if value is None: + return '' + if isinstance(value, list): + return '\n'.join(coerce_text(v) for v in value if coerce_text(v)) + if isinstance(value, dict): + if 'text' in value: + return coerce_text(value['text']) + if '@value' in value: + return coerce_text(value['@value']) + return json.dumps(value, ensure_ascii=False) + return unescape(str(value)).strip() + + +def slugify_tag(value: str) -> str: + value = value.strip().lower() + value = re.sub(r'[^a-z0-9]+', '-', value) + return value.strip('-')[:48] + + +def normalize_keywords(value: Any) -> list[str]: + out: list[str] = [] + if isinstance(value, str): + parts = re.split(r'[,;|]', value) + out.extend(p.strip() for p in parts if p.strip()) + elif isinstance(value, list): + for item in value: + out.extend(normalize_keywords(item)) + return out + + +def split_ingredient(line: str) -> tuple[str, str]: + original = ' '.join(line.strip().split()) + if not original: + return '', '' + cleaned = original.replace('–', '-').replace('—', '-') + tokens = cleaned.split() + desc_tokens: list[str] = [] + name_tokens = tokens[:] + + while name_tokens: + token = name_tokens[0].strip(',()').lower() + if re.fullmatch(r'[\d/.-]+', token) or token in {'x', '×'}: + desc_tokens.append(name_tokens.pop(0)) + continue + if token in UNIT_WORDS: + desc_tokens.append(name_tokens.pop(0)) + continue + if token in {'of'} and desc_tokens: + desc_tokens.append(name_tokens.pop(0)) + continue + break + + if not name_tokens: + return original[:128], '' + + while name_tokens and name_tokens[0].strip(',').lower() in STOPWORDS and len(name_tokens) > 1: + desc_tokens.append(name_tokens.pop(0)) + + name = ' '.join(name_tokens).strip(' ,;') + if ',' in name: + first, rest = [p.strip() for p in name.split(',', 1)] + if first and len(first.split()) <= 5: + name = first + if rest: + desc_tokens.append(rest) + description = ' '.join(desc_tokens).strip(' ,;') + return name[:128], description[:512] + + +def normalize_recipe(url: str, recipe_node: dict[str, Any], title_fallback: str | None = None) -> dict[str, Any]: + name = coerce_text(recipe_node.get('name')) or (title_fallback or url) + description_parts: list[str] = [] + desc = coerce_text(recipe_node.get('description')) + instructions = recipe_node.get('recipeInstructions') + instructions_text = coerce_text(instructions) + if desc: + description_parts.append(desc) + if instructions_text: + description_parts.append(instructions_text) + description = '\n\n'.join(part for part in description_parts if part).strip() + + ingredients = recipe_node.get('recipeIngredient') or [] + items = [] + seen_items: set[tuple[str, str]] = set() + for raw in ingredients: + line = coerce_text(raw) + if not line: + continue + item_name, item_desc = split_ingredient(line) + if not item_name: + item_name = line[:128] + key = (item_name.lower(), item_desc.lower()) + if key in seen_items: + continue + seen_items.add(key) + items.append({'name': item_name, 'description': item_desc, 'optional': False}) + + tags: list[str] = [] + for key in TAG_KEYS: + tags.extend(normalize_keywords(recipe_node.get(key))) + deduped_tags: list[str] = [] + seen_tags: set[str] = set() + for tag in tags: + slug = slugify_tag(tag) + if slug and slug not in seen_tags: + seen_tags.add(slug) + deduped_tags.append(slug) + + yields_val = recipe_node.get('recipeYield') + yields_num = None + if isinstance(yields_val, list) and yields_val: + yields_val = yields_val[0] + if yields_val is not None: + match = re.search(r'(\d+)', str(yields_val)) + if match: + yields_num = int(match.group(1)) + + payload = { + 'name': name[:128], + 'description': description[:20000], + 'source': url, + 'visibility': 0, + 'items': items, + 'tags': deduped_tags[:20], + } + cook_time = parse_duration_minutes(recipe_node.get('cookTime')) + prep_time = parse_duration_minutes(recipe_node.get('prepTime')) + total_time = parse_duration_minutes(recipe_node.get('totalTime')) + if cook_time is not None: + payload['cook_time'] = cook_time + if prep_time is not None: + payload['prep_time'] = prep_time + if total_time is not None: + payload['time'] = total_time + elif cook_time is not None or prep_time is not None: + payload['time'] = (cook_time or 0) + (prep_time or 0) + if yields_num is not None: + payload['yields'] = yields_num + return payload + + +def find_existing_recipe(base_url: str, token: str, household_id: int, payload: dict[str, Any]) -> dict[str, Any] | None: + status, data, _ = api_request('GET', f'{base_url}/api/household/{household_id}/recipe', token) + if status != 200 or not isinstance(data, list): + return None + want_source = (payload.get('source') or '').strip().lower() + want_name = (payload.get('name') or '').strip().lower() + for recipe in data: + source = str(recipe.get('source') or '').strip().lower() + name = str(recipe.get('name') or '').strip().lower() + if want_source and source and source == want_source: + return recipe + if want_name and name == want_name: + return recipe + return None + + +def import_one(args: argparse.Namespace, base_url: str, token: str, household_id: int, url: str) -> dict[str, Any]: + result: dict[str, Any] = {'url': url} + payload = None + scrape_error = None + + if not args.skip_kitchenowl_scrape: + status, data, _ = api_request('GET', f'{base_url}/api/household/{household_id}/recipe/scrape', token, params={'url': url}, timeout=args.timeout) + if status == 200 and isinstance(data, dict): + payload = data + result['strategy'] = 'kitchenowl-scrape' + else: + scrape_error = {'status': status, 'detail': data} + + if payload is None: + html = fetch_url(url, timeout=args.timeout) + recipe_node, title = parse_json_ld_recipe(html) + if not recipe_node: + raise RuntimeError(f'No schema.org Recipe JSON-LD found for {url}') + payload = normalize_recipe(url, recipe_node, title) + result['strategy'] = 'json-ld-fallback' + if scrape_error: + result['scrape_error'] = scrape_error + + payload['visibility'] = args.visibility + result['payload'] = payload + + existing = None if args.allow_duplicates else find_existing_recipe(base_url, token, household_id, payload) + if existing: + result['existing'] = {'id': existing.get('id'), 'name': existing.get('name'), 'source': existing.get('source')} + result['status'] = 'duplicate' + return result + + if args.create: + status, created, _ = api_request('POST', f'{base_url}/api/household/{household_id}/recipe', token, body=payload, timeout=args.timeout) + result['create_status'] = status + result['created'] = created + result['status'] = 'created' if status == 200 else 'error' + else: + result['status'] = 'dry-run' + return result + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description='Import recipe links into KitchenOwl with Doris-managed fallback parsing.') + p.add_argument('urls', nargs='+', help='Recipe URLs to normalize/import') + p.add_argument('--env-file', default=str(DEFAULT_ENV_PATH), help='Optional .env file to load before reading env vars') + p.add_argument('--base-url', default=None, help='KitchenOwl base URL, e.g. https://owl.paccoco.com') + p.add_argument('--token', default=None, help='KitchenOwl bearer token (prefer env var instead)') + p.add_argument('--household-id', type=int, default=None, help='KitchenOwl household id') + p.add_argument('--create', action='store_true', help='Create recipes in KitchenOwl. Default is dry-run only.') + p.add_argument('--allow-duplicates', action='store_true', help='Do not skip name/source duplicates.') + p.add_argument('--skip-kitchenowl-scrape', action='store_true', help='Go straight to Doris JSON-LD parsing.') + p.add_argument('--visibility', type=int, choices=[0,1,2], default=0, help='Recipe visibility: 0 private, 1 link, 2 public.') + p.add_argument('--pretty', action='store_true', help='Pretty-print JSON output') + p.add_argument('--timeout', type=int, default=DEFAULT_TIMEOUT, help='HTTP timeout in seconds') + return p.parse_args() + + +def main() -> int: + args = parse_args() + load_env_file(Path(args.env_file).expanduser()) + + base_url = (args.base_url or env('KITCHENOWL_BASE_URL') or env('KITCHENOWL_URL') or '').rstrip('/') + token = args.token or env('KITCHENOWL_API_TOKEN') or '' + household_raw = args.household_id or env('KITCHENOWL_HOUSEHOLD_ID') + + if not base_url: + print('Missing KitchenOwl base URL. Set KITCHENOWL_BASE_URL or KITCHENOWL_URL.', file=sys.stderr) + return 2 + if not token: + print('Missing KitchenOwl API token. Set KITCHENOWL_API_TOKEN.', file=sys.stderr) + return 2 + if not household_raw: + print('Missing KitchenOwl household id. Set KITCHENOWL_HOUSEHOLD_ID.', file=sys.stderr) + return 2 + + household_id = int(household_raw) + results = [] + failures = 0 + for url in args.urls: + try: + results.append(import_one(args, base_url, token, household_id, url)) + except Exception as exc: + failures += 1 + results.append({'url': url, 'status': 'error', 'error': str(exc)}) + + dump = json.dumps(results, ensure_ascii=False, indent=2 if args.pretty else None) + print(dump) + return 1 if failures else 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/docs/operations/KITCHENOWL_RECIPE_IMPORT.md b/docs/operations/KITCHENOWL_RECIPE_IMPORT.md new file mode 100644 index 0000000..009d1ea --- /dev/null +++ b/docs/operations/KITCHENOWL_RECIPE_IMPORT.md @@ -0,0 +1,137 @@ +# KitchenOwl Recipe Import (Doris-managed) + +## Why this exists + +KitchenOwl's built-in import/scrape flow is unreliable for the way John and Leanne actually send recipe links around. + +This repo-side helper gives Doris a safer path: + +1. try KitchenOwl's native `/recipe/scrape` endpoint for sites it understands +2. if that fails, fetch the page directly +3. extract schema.org `Recipe` JSON-LD +4. normalize ingredients/tags/timings into KitchenOwl's create-recipe payload +5. optionally create the recipe through the KitchenOwl API + +That means Doris can ingest recipe links from chat without depending on KitchenOwl's flaky native importer. + +## Script + +```bash +automation/bin/kitchenowl_recipe_import.py +``` + +Default mode is **dry-run**. It prints the normalized KitchenOwl payload and does **not** create anything unless `--create` is passed. + +## Required env vars + +Put these in a live `.env` file or export them ad hoc. Do **not** commit live secrets. + +```env +KITCHENOWL_BASE_URL=https://owl.paccoco.com +KITCHENOWL_API_TOKEN=CHANGE_ME +KITCHENOWL_HOUSEHOLD_ID=1 +``` + +Notes: +- `KITCHENOWL_API_TOKEN` should be a long-lived token for a user who can create recipes in the target household. +- `KITCHENOWL_HOUSEHOLD_ID` for the current live instance is `1` (`Stafford's`) unless that changes later. + +## Usage + +### Dry-run a link + +```bash +cd /home/fizzlepoof/repos/truenas-stacks +export KITCHENOWL_BASE_URL=https://owl.paccoco.com +export KITCHENOWL_API_TOKEN=...redacted... +export KITCHENOWL_HOUSEHOLD_ID=1 + +automation/bin/kitchenowl_recipe_import.py --pretty \ + "https://example.com/my-recipe" +``` + +### Actually create the recipe + +```bash +automation/bin/kitchenowl_recipe_import.py --pretty --create \ + "https://example.com/my-recipe" +``` + +### Import several links at once + +```bash +automation/bin/kitchenowl_recipe_import.py --create --pretty \ + "https://example.com/recipe-1" \ + "https://example.com/recipe-2" +``` + +## Behavior details + +### Strategy order + +By default the helper tries: + +1. `GET /api/household//recipe/scrape?url=...` +2. if KitchenOwl returns `Unsupported website` or otherwise fails, Doris falls back to direct HTML fetch + JSON-LD parse + +If you want to skip the built-in scrape and go straight to Doris mode: + +```bash +automation/bin/kitchenowl_recipe_import.py --skip-kitchenowl-scrape --pretty URL +``` + +### Duplicate protection + +Before creating a recipe, the helper loads existing household recipes and skips creation if it finds: +- an exact matching `source` URL, or +- an exact matching recipe `name` + +To force duplicates anyway: + +```bash +automation/bin/kitchenowl_recipe_import.py --create --allow-duplicates URL +``` + +### Visibility + +KitchenOwl visibility enum: +- `0` = private (default) +- `1` = link +- `2` = public + +Example: + +```bash +automation/bin/kitchenowl_recipe_import.py --create --visibility 0 URL +``` + +## What parses well + +The fallback parser works best on recipe pages that expose proper schema.org `Recipe` JSON-LD, which many decent recipe sites do. + +It pulls: +- name +- description +- instructions +- ingredient list +- total/prep/cook time +- yield/servings +- categories/cuisine/keywords → KitchenOwl tags + +## Known limitations + +- Some sites block scraping aggressively; Doris first tries normal fetch, then falls back to `curl` headers for better odds. +- Ingredient normalization is heuristic, not magic. It aims for a sane KitchenOwl item name plus a description/amount, but some weird ingredients may still need manual cleanup. +- Image upload/import is not wired yet in this helper. +- The helper expects structured JSON-LD. If a site hides the recipe entirely in client-side blobs or anti-bot nonsense, manual copy/paste may still be needed. +- `GET /api/settings` on the live KitchenOwl instance currently returns `500`, but that does not block recipe import. + +## Doris workflow expectation + +Preferred operator flow when John or Leanne sends recipe links: + +1. Doris runs a dry-run first when the site is unfamiliar or suspicious. +2. If the normalized payload looks sane, Doris reruns with `--create`. +3. If the page is too messy, Doris asks for the raw ingredients/steps and imports it manually. + +That keeps KitchenOwl as the recipe store while making Doris the reliable intake layer. diff --git a/docs/planning/TODO.md b/docs/planning/TODO.md index 4b964e5..1aeee96 100644 --- a/docs/planning/TODO.md +++ b/docs/planning/TODO.md @@ -1,27 +1,27 @@ # Homelab TODO ## Security (do ASAP) -- [ ] Regenerate Wings token on N.O.M.A.D. (was exposed in chat) -- [ ] Regenerate N.O.M.A.D. Newt secret (was exposed in chat) +- [x] Regenerate Wings token on N.O.M.A.D. (was exposed in chat) — completed 2026-05-15 +- [x] Regenerate N.O.M.A.D. Newt secret (was exposed in chat) — completed 2026-05-15 ## Infrastructure - [ ] Deploy Pi-hole 3-node HA cluster (PD + Serenity + RPi4) — see docs/planning/DEPLOY_PIHOLE.md - RPi4 IP still TBD — fill in before deploying -- [ ] Deploy Kima-Hub — see docs/planning/DEPLOY_KIMA_HUB.md +- [x] Deploy Kima-Hub — completed 2026-05-06; docs/planning/DEPLOY_KIMA_HUB.md now serves as deployment/repair reference - [ ] Set up off-site backup for vital data (appdata, databases, config repo, photos) - [ ] Remove dead Unraid-Cloudflared-Tunnel container from Serenity - [ ] Serenity malcolm pool capacity planning (heavily utilized) -- [ ] Add Pangolin resource for `ai.paccoco.com` → OpenWebUI (port 8282) -- [ ] Add Cloudflare DNS A record for `ai.paccoco.com` +- [x] Add Pangolin resource for `openwebui.paccoco.com` → OpenWebUI (port 8282) — completed 2026-05-15 +- [x] Add Cloudflare DNS record for `openwebui.paccoco.com` — completed 2026-05-15 - [ ] Install fresh editor on PD -- [ ] Set up private Gitea repo for .env file backups +- [x] Set up private Gitea repo for .env file backups — 2026-05-14 ## AI Stack -- [ ] Pull `qwen2.5:14b` on PD ollama: `sudo docker exec -it ollama ollama pull qwen2.5:14b` -- [ ] Create OpenWebUI account and set system prompt from HOMELAB_AI_CONTEXT.md -- [ ] Disable signups in OpenWebUI after creating account -- [ ] Connect Rocinante Ollama to OpenWebUI (Admin → Settings → Connections) -- [ ] Confirm `OLLAMA_HOST=0.0.0.0` set on Rocinante and Ollama service restarted +- [x] Pull `qwen2.5:14b` on PD ollama — completed 2026-05-15 +- [x] Create OpenWebUI account and set system prompt from HOMELAB_AI_CONTEXT.md — completed 2026-05-15 +- [x] Disable signups in OpenWebUI after creating account — completed 2026-05-15 +- [x] Connect Rocinante Ollama to OpenWebUI (Admin → Settings → Connections) — completed 2026-05-15 +- [x] Confirm `OLLAMA_HOST=0.0.0.0` set on Rocinante and Ollama service restarted — completed 2026-05-15 ## Healthchecks - [ ] Add healthcheck to ix-plex-plex-1 @@ -43,11 +43,12 @@ - [ ] Add remaining env vars to n8n docker-compose + .env: `PAPERLESS_API_TOKEN`, `PAPERLESS_TAG_TRANSCRIPTION`, `PAPERLESS_TAG_LECTURE_NOTES`, `N8N_API_KEY` (see n8n-workflows/README.md for docker-compose.yaml changes required) - [ ] Set up `PD SSH` credential in n8n for workflow 16 (see n8n-workflows/README.md) - [ ] Add sudoers NOPASSWD rule for n8n watchdog (see n8n-workflows/README.md) -- [ ] Import and activate upgraded workflows: 04, 08 (01 done — v1.4 active and tested) -- [ ] Import and activate new workflows: 14, 15 (16 already active) -- [ ] Configure class profiles + Paperless metadata mapping for current courses (ENGL-1010, HIST-2020, future classes) +- [x] Import and activate upgraded workflows: 04, 08 (01 done — v1.4 active and tested) — completed 2026-05-15 +- [x] Import and activate new workflows: 14, 15 (16 already active) — completed 2026-05-15 +- [x] Configure class profiles + Paperless metadata mapping for current courses (ENGL-1010, HIST-2020, future classes) — completed 2026-05-15 - [x] Delete legacy duplicate Paperless notes for school docs 42–49 after approval / with proper token context — completed 2026-05-14; docs now each have one canonical note -- [ ] Set up DB and Docker backups to Serenity following homelab SOP +- [x] Set up DB and Docker backups to Serenity following homelab SOP — completed 2026-05-15 + - Live `.env`, SSH trust, scheduler deployment, first run, and verification completed on PD ## Completed - [x] Deploy Gotify (Phase 1) — 2026-05-05 @@ -67,3 +68,4 @@ - [x] Workflow 01 fully tested end-to-end (dedup, LiteLLM, Gotify) — v1.4 active — 2026-05-09 - [x] Add LITELLM_API_KEY + N8N_BLOCK_ENV_ACCESS_IN_NODE to n8n docker-compose — 2026-05-09 - [x] Build Telegram chat-driven school-work intake for Paperless using shared Postgres, deterministic `intake_id`, and class/version metadata — 2026-05-14 +- [x] Build Doris-managed KitchenOwl recipe importer flow with native-scrape fallback, schema.org JSON-LD parsing, duplicate checks, and dry-run support — 2026-05-15