diff --git a/automation/bin/unifi_stage_hermes_dashboard_policy.py b/automation/bin/unifi_stage_hermes_dashboard_policy.py new file mode 100755 index 0000000..58a88b6 --- /dev/null +++ b/automation/bin/unifi_stage_hermes_dashboard_policy.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Stage/apply the narrow UniFi Policy Engine slice that makes the Hermes dashboard Trusted-only. + +Default mode is dry-run. Pass --apply to create/update the two desired policies. +Reads credentials from the PD runtime automation .env by default. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +from copy import deepcopy +from pathlib import Path +from typing import Any + +DEFAULT_RUNTIME_HELPER = Path('/mnt/docker-ssd/docker/compose/automation/bin/unifi_stage_low_risk_objects.py') +DEFAULT_ENV_PATH = Path('/mnt/docker-ssd/docker/compose/automation/.env') +INTERNAL_ZONE_NAME = 'Internal' + + +def load_helper(path: Path): + spec = importlib.util.spec_from_file_location('unifi_stage', path) + if spec is None or spec.loader is None: + raise RuntimeError(f'Could not load helper module from {path}') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def desired_policies() -> list[dict[str, Any]]: + return [ + { + 'name': 'Allow Trusted to Hermes Dashboard', + 'enabled': True, + 'action': 'ALLOW', + 'protocol': 'tcp', + 'ip_version': 'IPV4', + 'logging': False, + 'create_allow_respond': False, + 'connection_state_type': 'ALL', + 'connection_states': [], + 'match_ip_sec': False, + 'match_opposite_protocol': False, + 'icmp_typename': 'ANY', + 'icmp_v6_typename': 'ANY', + 'description': 'Allow Trusted clients to reach the Hermes dashboard on Nomad.', + 'source_zone_name': INTERNAL_ZONE_NAME, + 'destination_zone_name': INTERNAL_ZONE_NAME, + 'schedule': {'mode': 'ALWAYS'}, + 'source': { + 'matching_target': 'IP', + 'matching_target_type': 'SPECIFIC', + 'ips': ['10.5.1.0/24'], + 'match_mac': False, + 'match_opposite_ips': False, + 'match_opposite_ports': False, + 'port_matching_type': 'ANY', + }, + 'destination': { + 'matching_target': 'IP', + 'matching_target_type': 'SPECIFIC', + 'ips': ['10.5.30.7'], + 'match_opposite_ips': False, + 'match_opposite_ports': False, + 'port_matching_type': 'SPECIFIC', + 'port': '9119', + }, + }, + { + 'name': 'Block Management and Servers to Hermes Dashboard', + 'enabled': True, + 'action': 'BLOCK', + 'protocol': 'tcp', + 'ip_version': 'IPV4', + 'logging': False, + 'create_allow_respond': False, + 'connection_state_type': 'ALL', + 'connection_states': [], + 'match_ip_sec': False, + 'match_opposite_protocol': False, + 'icmp_typename': 'ANY', + 'icmp_v6_typename': 'ANY', + 'description': 'Block Management and Servers clients from reaching the Hermes dashboard on Nomad.', + 'source_zone_name': INTERNAL_ZONE_NAME, + 'destination_zone_name': INTERNAL_ZONE_NAME, + 'schedule': {'mode': 'ALWAYS'}, + 'source': { + 'matching_target': 'IP', + 'matching_target_type': 'SPECIFIC', + 'ips': ['10.5.0.0/24', '10.5.30.0/24'], + 'match_mac': False, + 'match_opposite_ips': False, + 'match_opposite_ports': False, + 'port_matching_type': 'ANY', + }, + 'destination': { + 'matching_target': 'IP', + 'matching_target_type': 'SPECIFIC', + 'ips': ['10.5.30.7'], + 'match_opposite_ips': False, + 'match_opposite_ports': False, + 'port_matching_type': 'SPECIFIC', + 'port': '9119', + }, + }, + ] + + +POLICY_MUTABLE_KEYS = { + 'name', + 'enabled', + 'action', + 'protocol', + 'ip_version', + 'logging', + 'create_allow_respond', + 'connection_state_type', + 'connection_states', + 'match_ip_sec', + 'match_opposite_protocol', + 'icmp_typename', + 'icmp_v6_typename', + 'source', + 'destination', + 'schedule', + 'description', +} + + +def canonical_policy(policy: dict[str, Any]) -> dict[str, Any]: + out = {key: deepcopy(policy.get(key)) for key in POLICY_MUTABLE_KEYS if key in policy} + out.setdefault('description', '') + out.setdefault('connection_state_type', 'ALL') + out.setdefault('connection_states', []) + out.setdefault('logging', False) + out.setdefault('match_ip_sec', False) + out.setdefault('match_opposite_protocol', False) + out.setdefault('icmp_typename', 'ANY') + out.setdefault('icmp_v6_typename', 'ANY') + out.setdefault('schedule', {'mode': 'ALWAYS'}) + return out + + +def resolve_zone_ids(zone_data: list[dict[str, Any]]) -> dict[str, str]: + by_name: dict[str, str] = {} + for zone in zone_data: + name = zone.get('name') + zid = zone.get('_id') + if name and zid: + by_name[name] = zid + return by_name + + +def materialize_policy(spec: dict[str, Any], zone_ids: dict[str, str]) -> dict[str, Any]: + source_zone = spec['source_zone_name'] + dest_zone = spec['destination_zone_name'] + if source_zone not in zone_ids: + raise KeyError(f'Missing source zone: {source_zone}') + if dest_zone not in zone_ids: + raise KeyError(f'Missing destination zone: {dest_zone}') + + policy = {k: deepcopy(v) for k, v in spec.items() if not k.endswith('_zone_name')} + policy['description'] = policy.get('description', '') + policy['source'] = deepcopy(policy['source']) + policy['destination'] = deepcopy(policy['destination']) + policy['source']['zone_id'] = zone_ids[source_zone] + policy['destination']['zone_id'] = zone_ids[dest_zone] + return policy + + +def main() -> int: + parser = argparse.ArgumentParser(description='Stage/apply the Hermes dashboard Trusted-only UniFi Policy Engine slice.') + parser.add_argument('--helper', default=str(DEFAULT_RUNTIME_HELPER), help='Path to runtime UniFi helper on PD') + parser.add_argument('--env-file', default=str(DEFAULT_ENV_PATH), help='Path to runtime .env on PD') + parser.add_argument('--site', default='default') + parser.add_argument('--apply', action='store_true') + args = parser.parse_args() + + helper = load_helper(Path(args.helper)) + helper.load_env_file(Path(args.env_file)) + client = helper.UniFiClient( + base_url=os.environ.get('UNIFI_BASE_URL', 'https://10.5.0.1'), + username=os.environ['UNIFI_USERNAME'], + password=os.environ['UNIFI_PASSWORD'], + verify_tls=helper.env_bool('UNIFI_VERIFY_TLS', False), + timeout=20, + ) + client.login() + + zones_payload = client.get(f'/proxy/network/v2/api/site/{args.site}/firewall/zone') + zone_data_raw = zones_payload['data'] if isinstance(zones_payload, dict) and 'data' in zones_payload else zones_payload + zone_data = zone_data_raw if isinstance(zone_data_raw, list) else [] + zone_ids = resolve_zone_ids(zone_data) + + policies_path = f'/proxy/network/v2/api/site/{args.site}/firewall-policies' + policies_payload = client.get(policies_path) + policy_data_raw = policies_payload['data'] if isinstance(policies_payload, dict) and 'data' in policies_payload else policies_payload + policy_data = policy_data_raw if isinstance(policy_data_raw, list) else [] + custom_by_name = {p.get('name'): p for p in policy_data if not p.get('predefined') and p.get('name')} + + result: dict[str, Any] = { + 'mode': 'apply' if args.apply else 'dry-run', + 'zones': sorted(zone_ids.keys()), + 'changes': [], + } + + for desired in desired_policies(): + materialized = materialize_policy(desired, zone_ids) + existing = custom_by_name.get(materialized['name']) + desired_canonical = canonical_policy(materialized) + if existing is None: + result['changes'].append({'name': materialized['name'], 'action': 'create', 'payload': desired_canonical}) + if args.apply: + status, payload = client._json_request('POST', policies_path, desired_canonical) + if status >= 300: + result['changes'][-1]['error'] = {'status': status, 'payload': payload} + continue + + existing_canonical = canonical_policy(existing) + if existing_canonical != desired_canonical: + update_payload = deepcopy(existing) + for key, value in desired_canonical.items(): + update_payload[key] = deepcopy(value) + result['changes'].append({ + 'name': materialized['name'], + 'action': 'update', + 'before': existing_canonical, + 'after': desired_canonical, + }) + if args.apply: + status, payload = client._json_request('PUT', f"{policies_path}/{existing['_id']}", update_payload) + if status >= 300: + result['changes'][-1]['error'] = {'status': status, 'payload': payload} + else: + result['changes'].append({'name': materialized['name'], 'action': 'noop'}) + + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/docs/troubleshooting/KNOWN_QUIRKS.md b/docs/troubleshooting/KNOWN_QUIRKS.md index 130caec..c2aaf5d 100644 --- a/docs/troubleshooting/KNOWN_QUIRKS.md +++ b/docs/troubleshooting/KNOWN_QUIRKS.md @@ -26,8 +26,15 @@ Per-service gotchas that aren't bugs but will bite you if you forget them. - Requires `/mnt/tank/docker/appdata/donetick/config/selfhosted.yaml` with full DB config - Uses viper config loader — `DT_ENV` controls config file path - Public exposure through Pangolin can stay unhealthy after renumbering if the target health-check hostname is stale even when the target IP/port are already fixed; verify both `ip` and `hcHostname` +- If PD-side Pangolin/Newt targets still probe a stale pre-renumbering host IP (for example `10.5.1.6`) but the backend is otherwise healthy, the fastest reversible recovery is a runtime `/32` compatibility alias on PD while the authoritative Pangolin target/health-check state is corrected +- If the PD `newt-loopback-bridge` helper is using `network_mode: "container:"`, restarting `ix-newt-newt-1` can leave the loopback relays broken until `newt-loopback-bridge` is also restarted; symptom is public 502/503 on `localhost`-backed Pangolin routes even after target health turns green - OIDC metadata for the frontend is exposed from `/api/v1/resource`; if the login button is missing, check that endpoint before debugging the SPA +### homepage +- Live config is `/mnt/tank/docker/appdata/homepage/services.yaml`; it is not currently repo-managed, so live edits should be mirrored back into docs when they matter operationally +- `books.paccoco.com` is the working public Calibre-Web route; `calibre.paccoco.com` / `kindle.paccoco.com` are legacy/broken unless separate Pangolin resources are created for them +- RoMm widget URLs must use a backend Homepage can actually reach from PD; if the only working path is the auth-gated public Pangolin route, remove the widget instead of leaving a stale literal LAN IP + ### shlink - Data directory must be `chmod 777` — runs as non-root user that doesn't match host default ownership diff --git a/home/doris-dashboard/docs/hermes-dashboard-trusted-access-apply-2026-05-24.md b/home/doris-dashboard/docs/hermes-dashboard-trusted-access-apply-2026-05-24.md new file mode 100644 index 0000000..c606c56 --- /dev/null +++ b/home/doris-dashboard/docs/hermes-dashboard-trusted-access-apply-2026-05-24.md @@ -0,0 +1,81 @@ +# Hermes dashboard Trusted-only Policy Engine apply — 2026-05-24 + +## Live change summary +Applied the narrow UniFi Policy Engine service slice that makes the Hermes dashboard on Nomad effectively Trusted-only. + +Dashboard endpoint: +- `10.5.30.7:9119` + +Apply time artifact stamp: +- `2026-05-24-061715` + +## What changed +Created these two custom firewall policies: + +1. `Allow Trusted to Hermes Dashboard` +- id: `6a1297eefb913dd84d4a3fbb` +- action: `ALLOW` +- protocol: `tcp` +- source: `10.5.1.0/24` +- destination: `10.5.30.7` +- destination port: `9119` +- schedule: `ALWAYS` + +2. `Block Management and Servers to Hermes Dashboard` +- id: `6a1297eefb913dd84d4a3fc0` +- action: `BLOCK` +- protocol: `tcp` +- source: `10.5.0.0/24`, `10.5.30.0/24` +- destination: `10.5.30.7` +- destination port: `9119` +- schedule: `ALWAYS` + +## Verification result +- Pre-apply baseline did not already contain either Hermes-dashboard policy. +- Post-apply readback shows both rules present as non-predefined custom policies. +- Apply helper returned no API errors. + +## Artifacts +Baselines and apply output saved in: +- `home/doris-dashboard/docs/baselines/unifi-firewall-policies-custom-2026-05-24-061715-pre-hermes-dashboard-slice.json` +- `home/doris-dashboard/docs/baselines/unifi-hermes-dashboard-policy-apply-2026-05-24-061715.json` +- `home/doris-dashboard/docs/baselines/unifi-firewall-policies-custom-2026-05-24-061715-post-hermes-dashboard-slice.json` + +Helper added to repo and copied to PD runtime for repeatable staging/apply: +- repo: `automation/bin/unifi_stage_hermes_dashboard_policy.py` +- PD runtime: `/mnt/docker-ssd/docker/compose/automation/bin/unifi_stage_hermes_dashboard_policy.py` + +## Representative lane probe after apply +Follow-up live check from PD showed an important enforcement caveat. + +From PD (`10.5.30.6`) to Nomad (`10.5.30.7:9119`): +- direct TCP connect succeeded +- HTTP GET succeeded with `200 OK` +- route on PD is on-link: `10.5.30.7 dev enp3s0f0 src 10.5.30.6` + +Interpretation: +- PD and Nomad are both on the `Servers` subnet (`10.5.30.0/24`) +- this flow stays L2-local and does not traverse the UniFi gateway +- therefore the gateway Policy Engine block does not stop PD from reaching the dashboard even though the custom rule exists + +What the current rule set still likely achieves: +- Trusted clients on `10.5.1.0/24` can reach the dashboard +- routed Management-lane access should still be governed by the custom block +- same-subnet Servers peers are not isolated by this gateway-only approach + +## Operator validation still worth doing +Because same-subnet east-west traffic bypasses the gateway enforcement point, the right checks are now: +- from a Trusted client: confirm `http://hermes.home.paccoco.com:9119` or the intended dashboard URL still loads +- from a Management client on a different subnet: confirm `10.5.30.7:9119` is blocked +- do not treat PD->Nomad reachability as proof the rule failed globally; it specifically proves intra-subnet server peers need host-local enforcement if they must be excluded + +## If true Trusted-only must exclude PD and other Servers peers +Use one of these instead of relying only on the gateway policy: +- add a host firewall on Nomad permitting only Trusted-source addresses/subnets to port `9119` +- move the dashboard to a lane where disallowed sources must cross a routed boundary +- use switch/ACL mechanisms that can enforce same-subnet isolation if available + +## Rollback +Fastest rollback is to delete the two Hermes-dashboard custom rules by id or rerun a small helper update that disables/removes them: +- `6a1297eefb913dd84d4a3fbb` +- `6a1297eefb913dd84d4a3fc0`