From d62a391cbfaadcfae348a9794a2a97e06123ef72 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Fri, 22 May 2026 21:23:15 +0000 Subject: [PATCH] Layer secret leak guardrails --- .githooks/pre-push | 18 ++ .github/workflows/secret-guardrails.yml | 33 ++++ docs/operations/SECRETS_MANAGEMENT.md | 23 ++- home/doris-dashboard/docs/baselines/README.md | 9 +- scripts/scan-secret-bearing-artifacts.sh | 159 +++++++++++++----- 5 files changed, 195 insertions(+), 47 deletions(-) create mode 100755 .githooks/pre-push create mode 100644 .github/workflows/secret-guardrails.yml diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..47d509d --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +scanner="$repo_root/scripts/scan-secret-bearing-artifacts.sh" + +while read -r local_ref local_sha remote_ref remote_sha; do + [[ -n "$local_sha" ]] || continue + if [[ "$local_sha" =~ ^0+$ ]]; then + continue + fi + + if [[ "$remote_sha" =~ ^0+$ ]]; then + "$scanner" --unpublished "$local_sha" + else + "$scanner" --git-range "$remote_sha..$local_sha" + fi +done diff --git a/.github/workflows/secret-guardrails.yml b/.github/workflows/secret-guardrails.yml new file mode 100644 index 0000000..71691e7 --- /dev/null +++ b/.github/workflows/secret-guardrails.yml @@ -0,0 +1,33 @@ +name: secret-guardrails + +on: + push: + pull_request: + +jobs: + artifact-secret-scan: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Make scanner executable + run: chmod +x scripts/scan-secret-bearing-artifacts.sh + + - name: Scan all tracked infra artifacts for embedded secrets + run: scripts/scan-secret-bearing-artifacts.sh --tracked + + gitleaks: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run gitleaks + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs/operations/SECRETS_MANAGEMENT.md b/docs/operations/SECRETS_MANAGEMENT.md index 888d358..62f6e13 100644 --- a/docs/operations/SECRETS_MANAGEMENT.md +++ b/docs/operations/SECRETS_MANAGEMENT.md @@ -239,27 +239,36 @@ See: - `docs/operations/INCIDENT_2026-05-22_UNIFI_BASELINE_SECRET_LEAK.md` - `home/doris-dashboard/docs/baselines/README.md` -### Pre-commit guardrail for infra artifacts +### Git hook + CI guardrails for infra artifacts -The repo now includes a narrow pre-commit scanner aimed specifically at catching secret-bearing controller/export artifacts before push. +The repo now includes layered guardrails aimed specifically at catching secret-bearing controller/export artifacts before they leave a workstation. Files: - `scripts/scan-secret-bearing-artifacts.sh` - `.githooks/pre-commit` +- `.githooks/pre-push` +- `.github/workflows/secret-guardrails.yml` -Enable it once per clone: +Enable the hooks once per clone: ```bash git config core.hooksPath .githooks -chmod +x .githooks/pre-commit scripts/scan-secret-bearing-artifacts.sh +chmod +x .githooks/pre-commit .githooks/pre-push scripts/scan-secret-bearing-artifacts.sh ``` -Run it manually on staged changes: +Manual runs: ```bash scripts/scan-secret-bearing-artifacts.sh --staged +scripts/scan-secret-bearing-artifacts.sh --tracked +scripts/scan-secret-bearing-artifacts.sh --git-range origin/main..HEAD ``` +What each layer does: +- `pre-commit`: scans staged files before a local commit is created +- `pre-push`: scans the commits about to be pushed, including unpublished branch history +- GitHub Actions: runs the narrow artifact scan across tracked files and also runs Gitleaks on pushes and pull requests + What it is for: - raw baselines - exports @@ -268,7 +277,9 @@ What it is for: - similar machine-generated infra artifacts What it is not: -- a full replacement for GitHub/GitGuardian or a generic secret scanner for every file type +- a full replacement for operator judgment about what belongs in the repo +- proof that a raw export is safe just because the scanner stayed quiet +- a reason to skip rotating live secrets if one ever does leak ## Security Reminders diff --git a/home/doris-dashboard/docs/baselines/README.md b/home/doris-dashboard/docs/baselines/README.md index 2ef67fc..1236c7e 100644 --- a/home/doris-dashboard/docs/baselines/README.md +++ b/home/doris-dashboard/docs/baselines/README.md @@ -29,8 +29,13 @@ See: 2. Inspect them for secret-bearing fields. 3. Commit only a markdown summary or an explicitly redacted artifact. 4. Prefer human-readable summaries over raw appliance dumps. -5. Enable the repo guardrail in each clone: +5. Enable the repo guardrails in each clone: ```bash git config core.hooksPath .githooks - chmod +x .githooks/pre-commit scripts/scan-secret-bearing-artifacts.sh + chmod +x .githooks/pre-commit .githooks/pre-push scripts/scan-secret-bearing-artifacts.sh + ``` +6. Before a risky push or cleanup branch, run the manual checks too: + ```bash + scripts/scan-secret-bearing-artifacts.sh --tracked + scripts/scan-secret-bearing-artifacts.sh --git-range origin/main..HEAD ``` diff --git a/scripts/scan-secret-bearing-artifacts.sh b/scripts/scan-secret-bearing-artifacts.sh index c1a785b..88dd72a 100755 --- a/scripts/scan-secret-bearing-artifacts.sh +++ b/scripts/scan-secret-bearing-artifacts.sh @@ -1,78 +1,159 @@ #!/usr/bin/env bash set -euo pipefail -mode="${1:---staged}" repo_root="$(git rev-parse --show-toplevel)" cd "$repo_root" -if [[ "$mode" != "--staged" && "$mode" != "--files" ]]; then - echo "usage: $0 [--staged|--files ...]" >&2 - exit 2 -fi - artifact_path_regex='(^|/)(docs/baselines/|baselines/|snapshots?/|exports?/|dumps?/|backups?/|diagnostics?/|artifacts?/|tmp/|temp/)|(^|/).*(baseline|snapshot|export|dump|diagnostic).*(\.json|\.ya?ml|\.txt|\.conf)?$' - placeholder_regex='(REDACTED|CHANGE_ME||example|placeholder|your[_ -]?token|your[_ -]?key)' +empty_tree='4b825dc642cb6eb9a060e54bf8d69288fbee4904' + +usage() { + cat >&2 <<'EOF' +usage: + scripts/scan-secret-bearing-artifacts.sh --staged + scripts/scan-secret-bearing-artifacts.sh --tracked + scripts/scan-secret-bearing-artifacts.sh --files ... + scripts/scan-secret-bearing-artifacts.sh --git-range + scripts/scan-secret-bearing-artifacts.sh --unpublished +EOF + exit 2 +} + +mode="${1:---staged}" +shift || true declare -a files=() -if [[ "$mode" == "--staged" ]]; then - while IFS= read -r f; do - [[ -n "$f" ]] && files+=("$f") - done < <(git diff --cached --name-only --diff-filter=ACMR) -else - shift - for f in "$@"; do - files+=("$f") - done -fi +declare -a commit_specs=() -if [[ ${#files[@]} -eq 0 ]]; then +case "$mode" in + --staged) + while IFS= read -r -d '' f; do + files+=("$f") + done < <(git diff --cached --name-only --diff-filter=ACMR -z) + ;; + --tracked) + while IFS= read -r -d '' f; do + files+=("$f") + done < <(git ls-files -z) + ;; + --files) + [[ $# -ge 1 ]] || usage + files=("$@") + ;; + --git-range) + [[ $# -eq 1 ]] || usage + while IFS= read -r commit; do + [[ -n "$commit" ]] && commit_specs+=("$commit") + done < <(git rev-list --reverse "$1") + ;; + --unpublished) + [[ $# -eq 1 ]] || usage + while IFS= read -r commit; do + [[ -n "$commit" ]] && commit_specs+=("$commit") + done < <(git rev-list --reverse "$1" --not --remotes) + ;; + *) + usage + ;; +esac + +if [[ ${#files[@]} -eq 0 && ${#commit_specs[@]} -eq 0 ]]; then exit 0 fi -failures=0 +python_scan() { + local display_name="$1" + local source_mode="$2" + local source_value="$3" -scan_file() { - local file="$1" - [[ -f "$file" ]] || return 0 - [[ "$file" =~ $artifact_path_regex ]] || return 0 - - python3 - "$file" "$placeholder_regex" <<'PY' + python3 - "$display_name" "$placeholder_regex" "$source_mode" "$source_value" <<'PY' from pathlib import Path -import re, sys +import re +import subprocess +import sys -path = Path(sys.argv[1]) +label = sys.argv[1] placeholder = re.compile(sys.argv[2], re.I) -text = path.read_text(errors='ignore') -lines = text.splitlines() +source_mode = sys.argv[3] +source_value = sys.argv[4] +if source_mode == 'file': + text = Path(source_value).read_text(errors='ignore') +else: + text = subprocess.check_output( + ['git', 'show', source_value], + text=True, + errors='ignore', + ) + +lines = text.splitlines() patterns = [ (re.compile(r'"x_wireguard_private_key"\s*:\s*"([^"]+)"', re.I), 'embedded WireGuard private key'), (re.compile(r'"wireguard_client_configuration_file"\s*:\s*"([^"\\]|\\.)*PrivateKey = [A-Za-z0-9+/=]{20,}', re.I), 'embedded WireGuard client config with private key'), (re.compile(r'PrivateKey\s*=\s*([A-Za-z0-9+/=]{20,})'), 'WireGuard private key line'), + (re.compile(r'BEGIN [A-Z0-9 ]*PRIVATE KEY', re.I), 'private key block header'), (re.compile(r'Authorization\s*[:=]\s*(Bearer\s+)?[A-Za-z0-9._~+/=-]{16,}', re.I), 'authorization header/token material'), (re.compile(r'"(token|api[_-]?key|client[_-]?secret|password|secret|cookie)"\s*:\s*"([^"]{12,})"', re.I), 'suspicious secret-like JSON field'), ] for idx, line in enumerate(lines, start=1): - for regex, label in patterns: - m = regex.search(line) - if not m: + for regex, finding in patterns: + match = regex.search(line) + if not match: continue if placeholder.search(line): continue - print(f'{path}:{idx}: {label}') + print(f'{label}:{idx}: {finding}') print(f' {line[:220]}') sys.exit(1) sys.exit(0) PY } -for file in "${files[@]}"; do - if ! scan_file "$file"; then - failures=1 - fi -done +scan_worktree_file() { + local file="$1" + [[ -f "$file" ]] || return 0 + [[ "$file" =~ $artifact_path_regex ]] || return 0 + python_scan "$file" file "$file" +} + +scan_commit_file() { + local commit="$1" + local file="$2" + git cat-file -e "$commit:$file" 2>/dev/null || return 0 + [[ "$file" =~ $artifact_path_regex ]] || return 0 + python_scan "$commit:$file" git_object "$commit:$file" +} + +failures=0 + +if [[ ${#files[@]} -gt 0 ]]; then + for file in "${files[@]}"; do + if ! scan_worktree_file "$file"; then + failures=1 + fi + done +fi + +if [[ ${#commit_specs[@]} -gt 0 ]]; then + declare -A seen=() + for commit in "${commit_specs[@]}"; do + parent="$empty_tree" + if git rev-parse --verify "$commit^" >/dev/null 2>&1; then + parent="$commit^" + fi + while IFS= read -r -d '' file; do + key="$commit:$file" + [[ -n "$file" ]] || continue + [[ -n "${seen[$key]:-}" ]] && continue + seen[$key]=1 + if ! scan_commit_file "$commit" "$file"; then + failures=1 + fi + done < <(git diff --name-only --diff-filter=ACMR -z "$parent" "$commit") + done +fi if [[ "$failures" -ne 0 ]]; then cat >&2 <<'EOF' @@ -80,7 +161,7 @@ if [[ "$failures" -ne 0 ]]; then Secret-bearing artifact scan failed. This guardrail is intentionally narrow: it is meant to stop raw controller/export/baseline artifacts -from being committed with embedded keys or tokens. +from being committed or pushed with embedded keys or tokens. Fix options: - remove the raw artifact from the commit