ops: record hermes dashboard trusted access slice

automation/bin/unifi_stage_hermes_dashboard_policy.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""Stage/apply the narrow UniFi Policy Engine slice that makes the Hermes dashboard Trusted-only.
+
+Default mode is dry-run. Pass --apply to create/update the two desired policies.
+Reads credentials from the PD runtime automation .env by default.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import os
+from copy import deepcopy
+from pathlib import Path
+from typing import Any
+
+DEFAULT_RUNTIME_HELPER = Path('/mnt/docker-ssd/docker/compose/automation/bin/unifi_stage_low_risk_objects.py')
+DEFAULT_ENV_PATH = Path('/mnt/docker-ssd/docker/compose/automation/.env')
+INTERNAL_ZONE_NAME = 'Internal'
+
+
+def load_helper(path: Path):
+    spec = importlib.util.spec_from_file_location('unifi_stage', path)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f'Could not load helper module from {path}')
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def desired_policies() -> list[dict[str, Any]]:
+    return [
+        {
+            'name': 'Allow Trusted to Hermes Dashboard',
+            'enabled': True,
+            'action': 'ALLOW',
+            'protocol': 'tcp',
+            'ip_version': 'IPV4',
+            'logging': False,
+            'create_allow_respond': False,
+            'connection_state_type': 'ALL',
+            'connection_states': [],
+            'match_ip_sec': False,
+            'match_opposite_protocol': False,
+            'icmp_typename': 'ANY',
+            'icmp_v6_typename': 'ANY',
+            'description': 'Allow Trusted clients to reach the Hermes dashboard on Nomad.',
+            'source_zone_name': INTERNAL_ZONE_NAME,
+            'destination_zone_name': INTERNAL_ZONE_NAME,
+            'schedule': {'mode': 'ALWAYS'},
+            'source': {
+                'matching_target': 'IP',
+                'matching_target_type': 'SPECIFIC',
+                'ips': ['10.5.1.0/24'],
+                'match_mac': False,
+                'match_opposite_ips': False,
+                'match_opposite_ports': False,
+                'port_matching_type': 'ANY',
+            },
+            'destination': {
+                'matching_target': 'IP',
+                'matching_target_type': 'SPECIFIC',
+                'ips': ['10.5.30.7'],
+                'match_opposite_ips': False,
+                'match_opposite_ports': False,
+                'port_matching_type': 'SPECIFIC',
+                'port': '9119',
+            },
+        },
+        {
+            'name': 'Block Management and Servers to Hermes Dashboard',
+            'enabled': True,
+            'action': 'BLOCK',
+            'protocol': 'tcp',
+            'ip_version': 'IPV4',
+            'logging': False,
+            'create_allow_respond': False,
+            'connection_state_type': 'ALL',
+            'connection_states': [],
+            'match_ip_sec': False,
+            'match_opposite_protocol': False,
+            'icmp_typename': 'ANY',
+            'icmp_v6_typename': 'ANY',
+            'description': 'Block Management and Servers clients from reaching the Hermes dashboard on Nomad.',
+            'source_zone_name': INTERNAL_ZONE_NAME,
+            'destination_zone_name': INTERNAL_ZONE_NAME,
+            'schedule': {'mode': 'ALWAYS'},
+            'source': {
+                'matching_target': 'IP',
+                'matching_target_type': 'SPECIFIC',
+                'ips': ['10.5.0.0/24', '10.5.30.0/24'],
+                'match_mac': False,
+                'match_opposite_ips': False,
+                'match_opposite_ports': False,
+                'port_matching_type': 'ANY',
+            },
+            'destination': {
+                'matching_target': 'IP',
+                'matching_target_type': 'SPECIFIC',
+                'ips': ['10.5.30.7'],
+                'match_opposite_ips': False,
+                'match_opposite_ports': False,
+                'port_matching_type': 'SPECIFIC',
+                'port': '9119',
+            },
+        },
+    ]
+
+
+POLICY_MUTABLE_KEYS = {
+    'name',
+    'enabled',
+    'action',
+    'protocol',
+    'ip_version',
+    'logging',
+    'create_allow_respond',
+    'connection_state_type',
+    'connection_states',
+    'match_ip_sec',
+    'match_opposite_protocol',
+    'icmp_typename',
+    'icmp_v6_typename',
+    'source',
+    'destination',
+    'schedule',
+    'description',
+}
+
+
+def canonical_policy(policy: dict[str, Any]) -> dict[str, Any]:
+    out = {key: deepcopy(policy.get(key)) for key in POLICY_MUTABLE_KEYS if key in policy}
+    out.setdefault('description', '')
+    out.setdefault('connection_state_type', 'ALL')
+    out.setdefault('connection_states', [])
+    out.setdefault('logging', False)
+    out.setdefault('match_ip_sec', False)
+    out.setdefault('match_opposite_protocol', False)
+    out.setdefault('icmp_typename', 'ANY')
+    out.setdefault('icmp_v6_typename', 'ANY')
+    out.setdefault('schedule', {'mode': 'ALWAYS'})
+    return out
+
+
+def resolve_zone_ids(zone_data: list[dict[str, Any]]) -> dict[str, str]:
+    by_name: dict[str, str] = {}
+    for zone in zone_data:
+        name = zone.get('name')
+        zid = zone.get('_id')
+        if name and zid:
+            by_name[name] = zid
+    return by_name
+
+
+def materialize_policy(spec: dict[str, Any], zone_ids: dict[str, str]) -> dict[str, Any]:
+    source_zone = spec['source_zone_name']
+    dest_zone = spec['destination_zone_name']
+    if source_zone not in zone_ids:
+        raise KeyError(f'Missing source zone: {source_zone}')
+    if dest_zone not in zone_ids:
+        raise KeyError(f'Missing destination zone: {dest_zone}')
+
+    policy = {k: deepcopy(v) for k, v in spec.items() if not k.endswith('_zone_name')}
+    policy['description'] = policy.get('description', '')
+    policy['source'] = deepcopy(policy['source'])
+    policy['destination'] = deepcopy(policy['destination'])
+    policy['source']['zone_id'] = zone_ids[source_zone]
+    policy['destination']['zone_id'] = zone_ids[dest_zone]
+    return policy
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description='Stage/apply the Hermes dashboard Trusted-only UniFi Policy Engine slice.')
+    parser.add_argument('--helper', default=str(DEFAULT_RUNTIME_HELPER), help='Path to runtime UniFi helper on PD')
+    parser.add_argument('--env-file', default=str(DEFAULT_ENV_PATH), help='Path to runtime .env on PD')
+    parser.add_argument('--site', default='default')
+    parser.add_argument('--apply', action='store_true')
+    args = parser.parse_args()
+
+    helper = load_helper(Path(args.helper))
+    helper.load_env_file(Path(args.env_file))
+    client = helper.UniFiClient(
+        base_url=os.environ.get('UNIFI_BASE_URL', 'https://10.5.0.1'),
+        username=os.environ['UNIFI_USERNAME'],
+        password=os.environ['UNIFI_PASSWORD'],
+        verify_tls=helper.env_bool('UNIFI_VERIFY_TLS', False),
+        timeout=20,
+    )
+    client.login()
+
+    zones_payload = client.get(f'/proxy/network/v2/api/site/{args.site}/firewall/zone')
+    zone_data_raw = zones_payload['data'] if isinstance(zones_payload, dict) and 'data' in zones_payload else zones_payload
+    zone_data = zone_data_raw if isinstance(zone_data_raw, list) else []
+    zone_ids = resolve_zone_ids(zone_data)
+
+    policies_path = f'/proxy/network/v2/api/site/{args.site}/firewall-policies'
+    policies_payload = client.get(policies_path)
+    policy_data_raw = policies_payload['data'] if isinstance(policies_payload, dict) and 'data' in policies_payload else policies_payload
+    policy_data = policy_data_raw if isinstance(policy_data_raw, list) else []
+    custom_by_name = {p.get('name'): p for p in policy_data if not p.get('predefined') and p.get('name')}
+
+    result: dict[str, Any] = {
+        'mode': 'apply' if args.apply else 'dry-run',
+        'zones': sorted(zone_ids.keys()),
+        'changes': [],
+    }
+
+    for desired in desired_policies():
+        materialized = materialize_policy(desired, zone_ids)
+        existing = custom_by_name.get(materialized['name'])
+        desired_canonical = canonical_policy(materialized)
+        if existing is None:
+            result['changes'].append({'name': materialized['name'], 'action': 'create', 'payload': desired_canonical})
+            if args.apply:
+                status, payload = client._json_request('POST', policies_path, desired_canonical)
+                if status >= 300:
+                    result['changes'][-1]['error'] = {'status': status, 'payload': payload}
+            continue
+
+        existing_canonical = canonical_policy(existing)
+        if existing_canonical != desired_canonical:
+            update_payload = deepcopy(existing)
+            for key, value in desired_canonical.items():
+                update_payload[key] = deepcopy(value)
+            result['changes'].append({
+                'name': materialized['name'],
+                'action': 'update',
+                'before': existing_canonical,
+                'after': desired_canonical,
+            })
+            if args.apply:
+                status, payload = client._json_request('PUT', f"{policies_path}/{existing['_id']}", update_payload)
+                if status >= 300:
+                    result['changes'][-1]['error'] = {'status': status, 'payload': payload}
+        else:
+            result['changes'].append({'name': materialized['name'], 'action': 'noop'})
+
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

docs/troubleshooting/KNOWN_QUIRKS.md

@@ -26,8 +26,15 @@ Per-service gotchas that aren't bugs but will bite you if you forget them.
 - Requires `/mnt/tank/docker/appdata/donetick/config/selfhosted.yaml` with full DB config
 - Uses viper config loader — `DT_ENV` controls config file path
 - Public exposure through Pangolin can stay unhealthy after renumbering if the target health-check hostname is stale even when the target IP/port are already fixed; verify both `ip` and `hcHostname`
+- If PD-side Pangolin/Newt targets still probe a stale pre-renumbering host IP (for example `10.5.1.6`) but the backend is otherwise healthy, the fastest reversible recovery is a runtime `/32` compatibility alias on PD while the authoritative Pangolin target/health-check state is corrected
+- If the PD `newt-loopback-bridge` helper is using `network_mode: "container:<newt>"`, restarting `ix-newt-newt-1` can leave the loopback relays broken until `newt-loopback-bridge` is also restarted; symptom is public 502/503 on `localhost`-backed Pangolin routes even after target health turns green
 - OIDC metadata for the frontend is exposed from `/api/v1/resource`; if the login button is missing, check that endpoint before debugging the SPA
 
+### homepage
+- Live config is `/mnt/tank/docker/appdata/homepage/services.yaml`; it is not currently repo-managed, so live edits should be mirrored back into docs when they matter operationally
+- `books.paccoco.com` is the working public Calibre-Web route; `calibre.paccoco.com` / `kindle.paccoco.com` are legacy/broken unless separate Pangolin resources are created for them
+- RoMm widget URLs must use a backend Homepage can actually reach from PD; if the only working path is the auth-gated public Pangolin route, remove the widget instead of leaving a stale literal LAN IP
+
 ### shlink
 - Data directory must be `chmod 777` — runs as non-root user that doesn't match host default ownership
 

home/doris-dashboard/docs/hermes-dashboard-trusted-access-apply-2026-05-24.md

@@ -0,0 +1,81 @@
+# Hermes dashboard Trusted-only Policy Engine apply — 2026-05-24
+
+## Live change summary
+Applied the narrow UniFi Policy Engine service slice that makes the Hermes dashboard on Nomad effectively Trusted-only.
+
+Dashboard endpoint:
+- `10.5.30.7:9119`
+
+Apply time artifact stamp:
+- `2026-05-24-061715`
+
+## What changed
+Created these two custom firewall policies:
+
+1. `Allow Trusted to Hermes Dashboard`
+- id: `6a1297eefb913dd84d4a3fbb`
+- action: `ALLOW`
+- protocol: `tcp`
+- source: `10.5.1.0/24`
+- destination: `10.5.30.7`
+- destination port: `9119`
+- schedule: `ALWAYS`
+
+2. `Block Management and Servers to Hermes Dashboard`
+- id: `6a1297eefb913dd84d4a3fc0`
+- action: `BLOCK`
+- protocol: `tcp`
+- source: `10.5.0.0/24`, `10.5.30.0/24`
+- destination: `10.5.30.7`
+- destination port: `9119`
+- schedule: `ALWAYS`
+
+## Verification result
+- Pre-apply baseline did not already contain either Hermes-dashboard policy.
+- Post-apply readback shows both rules present as non-predefined custom policies.
+- Apply helper returned no API errors.
+
+## Artifacts
+Baselines and apply output saved in:
+- `home/doris-dashboard/docs/baselines/unifi-firewall-policies-custom-2026-05-24-061715-pre-hermes-dashboard-slice.json`
+- `home/doris-dashboard/docs/baselines/unifi-hermes-dashboard-policy-apply-2026-05-24-061715.json`
+- `home/doris-dashboard/docs/baselines/unifi-firewall-policies-custom-2026-05-24-061715-post-hermes-dashboard-slice.json`
+
+Helper added to repo and copied to PD runtime for repeatable staging/apply:
+- repo: `automation/bin/unifi_stage_hermes_dashboard_policy.py`
+- PD runtime: `/mnt/docker-ssd/docker/compose/automation/bin/unifi_stage_hermes_dashboard_policy.py`
+
+## Representative lane probe after apply
+Follow-up live check from PD showed an important enforcement caveat.
+
+From PD (`10.5.30.6`) to Nomad (`10.5.30.7:9119`):
+- direct TCP connect succeeded
+- HTTP GET succeeded with `200 OK`
+- route on PD is on-link: `10.5.30.7 dev enp3s0f0 src 10.5.30.6`
+
+Interpretation:
+- PD and Nomad are both on the `Servers` subnet (`10.5.30.0/24`)
+- this flow stays L2-local and does not traverse the UniFi gateway
+- therefore the gateway Policy Engine block does not stop PD from reaching the dashboard even though the custom rule exists
+
+What the current rule set still likely achieves:
+- Trusted clients on `10.5.1.0/24` can reach the dashboard
+- routed Management-lane access should still be governed by the custom block
+- same-subnet Servers peers are not isolated by this gateway-only approach
+
+## Operator validation still worth doing
+Because same-subnet east-west traffic bypasses the gateway enforcement point, the right checks are now:
+- from a Trusted client: confirm `http://hermes.home.paccoco.com:9119` or the intended dashboard URL still loads
+- from a Management client on a different subnet: confirm `10.5.30.7:9119` is blocked
+- do not treat PD->Nomad reachability as proof the rule failed globally; it specifically proves intra-subnet server peers need host-local enforcement if they must be excluded
+
+## If true Trusted-only must exclude PD and other Servers peers
+Use one of these instead of relying only on the gateway policy:
+- add a host firewall on Nomad permitting only Trusted-source addresses/subnets to port `9119`
+- move the dashboard to a lane where disallowed sources must cross a routed boundary
+- use switch/ACL mechanisms that can enforce same-subnet isolation if available
+
+## Rollback
+Fastest rollback is to delete the two Hermes-dashboard custom rules by id or rerun a small helper update that disables/removes them:
+- `6a1297eefb913dd84d4a3fbb`
+- `6a1297eefb913dd84d4a3fc0`