4acc202c49
- scripts/setup-secrets-repo.sh: replace dpkg-deb install with Docker method - dev/docker-compose.yaml: add git-crypt-init service (profile: setup) - docs/operations/SECRETS_MANAGEMENT.md: document git-crypt encrypted repo, unlock procedure, rebuild runbook, and TrueNAS install methods
571 lines
18 KiB
Bash
Executable File
571 lines
18 KiB
Bash
Executable File
#!/bin/sh
|
|
# MeshMonitor Upgrade Watchdog
|
|
# Monitors for upgrade trigger file and performs Docker container upgrade
|
|
|
|
set -e
|
|
|
|
# Configuration
|
|
TRIGGER_FILE="${TRIGGER_FILE:-/data/.upgrade-trigger}"
|
|
STATUS_FILE="${STATUS_FILE:-/data/.upgrade-status}"
|
|
BACKUP_DIR="${BACKUP_DIR:-/data/backups}"
|
|
CHECK_INTERVAL="${CHECK_INTERVAL:-5}"
|
|
CONTAINER_NAME="${CONTAINER_NAME:-meshmonitor}"
|
|
IMAGE_NAME="${IMAGE_NAME:-ghcr.io/yeraze/meshmonitor}"
|
|
COMPOSE_PROJECT_DIR="${COMPOSE_PROJECT_DIR:-/compose}"
|
|
DOCKER_SOCKET_TEST_REQUEST="${DOCKER_SOCKET_TEST_REQUEST:-/data/.docker-socket-test-request}"
|
|
DOCKER_SOCKET_TEST_SCRIPT="${DOCKER_SOCKET_TEST_SCRIPT:-/data/.meshmonitor-internal/test-docker-socket.sh}"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
log() {
|
|
echo "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} ✅ $1"
|
|
}
|
|
|
|
log_warn() {
|
|
echo "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} ⚠️ $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo "${RED}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} ❌ $1"
|
|
}
|
|
|
|
# Write status to file for backend to read
|
|
write_status() {
|
|
echo "$1" > "$STATUS_FILE"
|
|
log "Status: $1"
|
|
}
|
|
|
|
# Create backup of data directory
|
|
create_backup() {
|
|
local backup_name="upgrade-backup-$(date +%Y%m%d_%H%M%S)"
|
|
local backup_path="$BACKUP_DIR/$backup_name"
|
|
|
|
log "Creating backup: $backup_path"
|
|
|
|
# Verify /data directory exists and is accessible
|
|
if [ ! -d "/data" ]; then
|
|
log_error "/data directory does not exist"
|
|
return 1
|
|
fi
|
|
|
|
# Create backup directory with error checking
|
|
if ! mkdir -p "$BACKUP_DIR" 2>/tmp/backup-error.log; then
|
|
log_error "Failed to create backup directory: $BACKUP_DIR"
|
|
if [ -f /tmp/backup-error.log ]; then
|
|
log_error "Error: $(cat /tmp/backup-error.log)"
|
|
rm -f /tmp/backup-error.log
|
|
fi
|
|
return 1
|
|
fi
|
|
|
|
# Verify backup directory was created
|
|
if [ ! -d "$BACKUP_DIR" ]; then
|
|
log_error "Backup directory does not exist after mkdir: $BACKUP_DIR"
|
|
return 1
|
|
fi
|
|
|
|
# Log directory info for debugging
|
|
log "Backup directory info: $(ls -ld "$BACKUP_DIR" 2>&1 || echo 'unable to stat')"
|
|
|
|
# Create backup (exclude backups directory itself)
|
|
# Capture stderr to a temp file for better error reporting
|
|
# Note: tar may report warnings about files disappearing (e.g., SQLite journal files)
|
|
# which can be safely ignored as long as the backup file is created successfully
|
|
tar -czf "$backup_path.tar.gz" -C /data --exclude='backups' --exclude='.upgrade-*' . 2>/tmp/tar-error.log
|
|
local tar_exit=$?
|
|
|
|
# Check if backup file was created and has content
|
|
if [ ! -f "$backup_path.tar.gz" ]; then
|
|
log_error "Backup file was not created: $backup_path.tar.gz"
|
|
|
|
# Show tar errors
|
|
if [ -f /tmp/tar-error.log ]; then
|
|
log_error "tar error output:"
|
|
while IFS= read -r line; do
|
|
log_error " $line"
|
|
done < /tmp/tar-error.log
|
|
fi
|
|
|
|
rm -f /tmp/tar-error.log
|
|
return 1
|
|
fi
|
|
|
|
local backup_size=$(stat -c%s "$backup_path.tar.gz" 2>/dev/null || echo "0")
|
|
|
|
# Check if backup is too small (likely corrupt)
|
|
if [ "$backup_size" -lt 100 ]; then
|
|
log_error "Backup file is suspiciously small (${backup_size} bytes)"
|
|
rm -f /tmp/tar-error.log
|
|
return 1
|
|
fi
|
|
|
|
# Exit code 0 = success, 1 = warnings (e.g., file changed during read)
|
|
# Both are acceptable as long as the backup file exists and has reasonable size
|
|
if [ $tar_exit -eq 0 ] || [ $tar_exit -eq 1 ]; then
|
|
if [ $tar_exit -eq 1 ] && [ -f /tmp/tar-error.log ]; then
|
|
# Log warnings but don't fail
|
|
log_warn "tar reported warnings (likely ephemeral files like SQLite journals):"
|
|
while IFS= read -r line; do
|
|
log_warn " $line"
|
|
done < /tmp/tar-error.log
|
|
fi
|
|
|
|
log_success "Backup created: $backup_path.tar.gz (${backup_size} bytes)"
|
|
rm -f /tmp/tar-error.log
|
|
echo "$backup_path.tar.gz"
|
|
return 0
|
|
else
|
|
log_error "Failed to create backup (tar exit code: $tar_exit)"
|
|
|
|
# Show the actual tar error
|
|
if [ -f /tmp/tar-error.log ]; then
|
|
log_error "tar error output:"
|
|
while IFS= read -r line; do
|
|
log_error " $line"
|
|
done < /tmp/tar-error.log
|
|
fi
|
|
|
|
rm -f /tmp/tar-error.log
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Pull new Docker image
|
|
pull_image() {
|
|
local version="$1"
|
|
local image="${IMAGE_NAME}:${version}"
|
|
|
|
log "Pulling image: $image"
|
|
|
|
if docker pull "$image"; then
|
|
log_success "Image pulled: $image"
|
|
|
|
# Tag as latest if not already latest
|
|
if [ "$version" != "latest" ]; then
|
|
docker tag "$image" "${IMAGE_NAME}:latest"
|
|
log_success "Tagged as latest"
|
|
fi
|
|
|
|
return 0
|
|
else
|
|
log_error "Failed to pull image: $image"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Recreate container with new image
|
|
recreate_container() {
|
|
log "Recreating container: $CONTAINER_NAME"
|
|
|
|
# Resolve compose directory - try configured path, then known mount points
|
|
# Older sidecar configs used /data/compose, current uses /compose
|
|
# Accept both .yml and .yaml extensions (both are valid for Docker Compose)
|
|
local compose_dir=""
|
|
local compose_file=""
|
|
for try_dir in "$COMPOSE_PROJECT_DIR" "/compose" "/data/compose"; do
|
|
if [ -d "$try_dir" ]; then
|
|
if [ -f "$try_dir/docker-compose.yml" ]; then
|
|
compose_dir="$try_dir"
|
|
compose_file="docker-compose.yml"
|
|
break
|
|
elif [ -f "$try_dir/docker-compose.yaml" ]; then
|
|
compose_dir="$try_dir"
|
|
compose_file="docker-compose.yaml"
|
|
break
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ -n "$compose_dir" ] && [ "$compose_dir" != "$COMPOSE_PROJECT_DIR" ]; then
|
|
if [ "$compose_dir" = "/data/compose" ]; then
|
|
log_warn "Falling back to legacy path /data/compose - please update COMPOSE_PROJECT_DIR to /compose"
|
|
else
|
|
log_warn "COMPOSE_PROJECT_DIR=$COMPOSE_PROJECT_DIR not found, falling back to $compose_dir"
|
|
fi
|
|
fi
|
|
|
|
if [ -z "$compose_dir" ]; then
|
|
log_error "No docker-compose.yml/yaml found at $COMPOSE_PROJECT_DIR, /compose, or /data/compose"
|
|
log_error "The upgrade sidecar requires Docker Compose files to recreate containers safely."
|
|
log_error "Mount your compose directory to /compose in the sidecar container."
|
|
return 1
|
|
fi
|
|
|
|
log "Using compose directory: $compose_dir"
|
|
|
|
# Verify docker compose is available
|
|
if ! docker compose version >/dev/null 2>&1; then
|
|
log_error "docker compose not available - required for container recreation"
|
|
return 1
|
|
fi
|
|
|
|
# Detect which compose files were originally used
|
|
local original_config_files=$(docker inspect --format='{{index .Config.Labels "com.docker.compose.project.config_files"}}' "$CONTAINER_NAME" 2>/dev/null || echo "")
|
|
|
|
local compose_files=""
|
|
if [ -n "$original_config_files" ]; then
|
|
log "Original compose files: $original_config_files"
|
|
for config_file in $(echo "$original_config_files" | tr ',' ' '); do
|
|
local filename=$(basename "$config_file")
|
|
if [ -f "$compose_dir/$filename" ]; then
|
|
compose_files="$compose_files -f $filename"
|
|
log " Using: $filename"
|
|
else
|
|
log_warn " Not found: $filename (skipping)"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Fallback if no compose files detected from labels
|
|
if [ -z "$compose_files" ]; then
|
|
compose_files="-f $compose_file"
|
|
# Check for upgrade overlay with either extension
|
|
if [ -f "$compose_dir/docker-compose.upgrade.yml" ]; then
|
|
compose_files="$compose_files -f docker-compose.upgrade.yml"
|
|
elif [ -f "$compose_dir/docker-compose.upgrade.yaml" ]; then
|
|
compose_files="$compose_files -f docker-compose.upgrade.yaml"
|
|
fi
|
|
log "Using default compose files: $compose_files"
|
|
fi
|
|
|
|
# Detect project name from container labels
|
|
local detected_project=$(docker inspect --format='{{index .Config.Labels "com.docker.compose.project"}}' "$CONTAINER_NAME" 2>/dev/null || echo "")
|
|
local project_flag=""
|
|
if [ -n "$detected_project" ]; then
|
|
project_flag="-p $detected_project"
|
|
log "Using project name: $detected_project"
|
|
elif [ -n "$COMPOSE_PROJECT_NAME" ]; then
|
|
project_flag="-p $COMPOSE_PROJECT_NAME"
|
|
log "Using env project name: $COMPOSE_PROJECT_NAME"
|
|
fi
|
|
|
|
# Pull latest image
|
|
log "Pulling latest image..."
|
|
if docker pull "${IMAGE_NAME}:latest" 2>/dev/null; then
|
|
log_success "Image pulled: ${IMAGE_NAME}:latest"
|
|
fi
|
|
|
|
# Recreate via docker compose (handles ports, volumes, env, networks - everything)
|
|
cd "$compose_dir" || return 1
|
|
log "Running: docker compose $project_flag $compose_files up -d --force-recreate --no-deps $CONTAINER_NAME"
|
|
|
|
if docker compose $project_flag $compose_files up -d --force-recreate --no-deps "$CONTAINER_NAME"; then
|
|
# Log the port mappings on the new container for verification
|
|
local new_ports=$(docker port "$CONTAINER_NAME" 2>/dev/null)
|
|
if [ -n "$new_ports" ]; then
|
|
log "Port mappings on recreated container:"
|
|
echo "$new_ports" | while IFS= read -r line; do log " $line"; done
|
|
fi
|
|
log_success "Container recreated successfully via Docker Compose"
|
|
return 0
|
|
else
|
|
log_error "Failed to recreate container via Docker Compose"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Wait for container health check
|
|
wait_for_health() {
|
|
local max_wait=120
|
|
local elapsed=0
|
|
|
|
log "Waiting for container health check..."
|
|
|
|
# Get BASE_URL from container env vars, default to empty if not set
|
|
local base_url=$(docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$CONTAINER_NAME" 2>/dev/null | grep '^BASE_URL=' | cut -d'=' -f2 | tr -d '\r\n')
|
|
|
|
# Construct health endpoint URL with BASE_URL if present
|
|
local health_path="/api/health"
|
|
if [ -n "$base_url" ] && [ "$base_url" != "/" ]; then
|
|
health_path="${base_url}/api/health"
|
|
fi
|
|
|
|
while [ $elapsed -lt $max_wait ]; do
|
|
# Check if container is running
|
|
if ! docker ps --filter "name=$CONTAINER_NAME" --filter "status=running" | grep -q "$CONTAINER_NAME"; then
|
|
log_warn "Container not running yet..."
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
continue
|
|
fi
|
|
|
|
# Get container IP directly from Docker inspect - more reliable than DNS after recreation
|
|
# Try multiple networks as the container might be on different networks
|
|
local container_ip=""
|
|
container_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_NAME" 2>/dev/null | head -n1)
|
|
|
|
# If no IP yet, wait for container to get network assigned
|
|
if [ -z "$container_ip" ]; then
|
|
log_warn "Container has no IP assigned yet..."
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
continue
|
|
fi
|
|
|
|
local health_url="http://${container_ip}:3001${health_path}"
|
|
|
|
# Only log URL on first attempt or if it changed
|
|
if [ "$elapsed" -eq 0 ] || [ -z "$last_health_url" ] || [ "$health_url" != "$last_health_url" ]; then
|
|
log "Health endpoint: $health_url"
|
|
last_health_url="$health_url"
|
|
fi
|
|
|
|
# Try to check health endpoint using container IP directly
|
|
# This is more reliable than container name DNS after recreation
|
|
if wget -q -O /dev/null --timeout=5 "$health_url" 2>/dev/null || \
|
|
curl -sf "$health_url" >/dev/null 2>&1; then
|
|
log_success "Health check passed"
|
|
return 0
|
|
fi
|
|
|
|
log "Waiting for health check... (${elapsed}s/${max_wait}s)"
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
|
|
log_error "Health check timeout after ${max_wait}s"
|
|
return 1
|
|
}
|
|
|
|
# Clean up old Docker images to free disk space
|
|
cleanup_old_images() {
|
|
log "Cleaning up old MeshMonitor images..."
|
|
|
|
# Get the current image ID being used by the container
|
|
local current_image_id=$(docker inspect --format='{{.Image}}' "$CONTAINER_NAME" 2>/dev/null | cut -d':' -f2 | cut -c1-12)
|
|
|
|
if [ -z "$current_image_id" ]; then
|
|
log_warn "Could not determine current image ID, skipping cleanup"
|
|
return 0
|
|
fi
|
|
|
|
log "Current image ID: $current_image_id"
|
|
|
|
# Find all MeshMonitor images (excluding the current one)
|
|
local old_images=$(docker images "${IMAGE_NAME}" --format '{{.ID}} {{.Tag}}' 2>/dev/null | while read id tag; do
|
|
# Get short ID for comparison
|
|
local short_id=$(echo "$id" | cut -c1-12)
|
|
if [ "$short_id" != "$current_image_id" ]; then
|
|
echo "$id"
|
|
fi
|
|
done)
|
|
|
|
if [ -z "$old_images" ]; then
|
|
log "No old images to clean up"
|
|
return 0
|
|
fi
|
|
|
|
# Count images to be removed
|
|
local image_count=$(echo "$old_images" | wc -l)
|
|
log "Found $image_count old image(s) to remove"
|
|
|
|
# Remove each old image
|
|
local removed=0
|
|
local failed=0
|
|
for image_id in $old_images; do
|
|
if docker rmi "$image_id" 2>/dev/null; then
|
|
removed=$((removed + 1))
|
|
log "Removed image: $image_id"
|
|
else
|
|
failed=$((failed + 1))
|
|
log_warn "Could not remove image: $image_id (may be in use)"
|
|
fi
|
|
done
|
|
|
|
if [ $removed -gt 0 ]; then
|
|
log_success "Cleaned up $removed old image(s)"
|
|
fi
|
|
|
|
if [ $failed -gt 0 ]; then
|
|
log_warn "$failed image(s) could not be removed"
|
|
fi
|
|
|
|
# Also clean up dangling images (untagged images from the build process)
|
|
local dangling=$(docker images -f "dangling=true" -q 2>/dev/null)
|
|
if [ -n "$dangling" ]; then
|
|
log "Removing dangling images..."
|
|
echo "$dangling" | xargs docker rmi 2>/dev/null || true
|
|
log_success "Dangling images cleaned up"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Perform upgrade
|
|
perform_upgrade() {
|
|
local trigger_data
|
|
local version
|
|
local backup_enabled
|
|
local upgrade_id
|
|
local backup_path
|
|
|
|
# Read trigger file
|
|
if [ ! -f "$TRIGGER_FILE" ]; then
|
|
log_error "Trigger file not found"
|
|
return 1
|
|
fi
|
|
|
|
trigger_data=$(cat "$TRIGGER_FILE")
|
|
version=$(echo "$trigger_data" | grep -o '"version"[ ]*:[ ]*"[^"]*"' | sed 's/.*:.*"\(.*\)"/\1/')
|
|
backup_enabled=$(echo "$trigger_data" | grep -o '"backup"[ ]*:[ ]*[^,}]*' | sed 's/.*:[ ]*//')
|
|
upgrade_id=$(echo "$trigger_data" | grep -o '"upgradeId"[ ]*:[ ]*"[^"]*"' | sed 's/.*:.*"\(.*\)"/\1/')
|
|
|
|
if [ -z "$version" ]; then
|
|
version="latest"
|
|
fi
|
|
|
|
log "=================================================="
|
|
log "Starting upgrade to version: $version"
|
|
log "Upgrade ID: $upgrade_id"
|
|
log "Backup enabled: $backup_enabled"
|
|
log "=================================================="
|
|
|
|
# Remove trigger file immediately to prevent re-triggering
|
|
rm -f "$TRIGGER_FILE"
|
|
|
|
# Step 1: Create backup
|
|
if [ "$backup_enabled" != "false" ]; then
|
|
write_status "backing_up"
|
|
if backup_path=$(create_backup); then
|
|
log_success "Backup completed: $backup_path"
|
|
else
|
|
write_status "failed"
|
|
log_error "Backup failed - aborting upgrade"
|
|
return 1
|
|
fi
|
|
else
|
|
log_warn "Backup disabled - skipping"
|
|
fi
|
|
|
|
# Step 2: Pull new image
|
|
write_status "downloading"
|
|
if ! pull_image "$version"; then
|
|
write_status "failed"
|
|
log_error "Image pull failed - aborting upgrade"
|
|
return 1
|
|
fi
|
|
|
|
# Step 3: Recreate container
|
|
write_status "restarting"
|
|
if ! recreate_container; then
|
|
write_status "failed"
|
|
log_error "Container recreation failed"
|
|
|
|
# Attempt rollback if backup exists
|
|
if [ -n "$backup_path" ] && [ -f "$backup_path" ]; then
|
|
log_warn "Attempting rollback..."
|
|
write_status "rolling_back"
|
|
# Rollback logic would go here
|
|
# For now, just log the error
|
|
log_error "Manual intervention required - backup available at: $backup_path"
|
|
fi
|
|
|
|
return 1
|
|
fi
|
|
|
|
# Step 4: Health check
|
|
write_status "health_check"
|
|
if ! wait_for_health; then
|
|
write_status "failed"
|
|
log_error "Health check failed - upgrade may have issues"
|
|
return 1
|
|
fi
|
|
|
|
# Step 5: Clean up old images to free disk space
|
|
write_status "cleanup"
|
|
cleanup_old_images
|
|
|
|
# Success!
|
|
write_status "complete"
|
|
log_success "=================================================="
|
|
log_success "Upgrade completed successfully!"
|
|
log_success "Version: $version"
|
|
log_success "Upgrade ID: $upgrade_id"
|
|
log_success "=================================================="
|
|
|
|
return 0
|
|
}
|
|
|
|
# Main loop
|
|
main() {
|
|
log "=================================================="
|
|
log "MeshMonitor Upgrade Watchdog Starting"
|
|
log "=================================================="
|
|
log "Container: $CONTAINER_NAME"
|
|
log "Image: $IMAGE_NAME"
|
|
log "Trigger file: $TRIGGER_FILE"
|
|
log "Check interval: ${CHECK_INTERVAL}s"
|
|
log "Compose project: $COMPOSE_PROJECT_DIR"
|
|
log "=================================================="
|
|
|
|
# Warn if running from the legacy script path
|
|
SCRIPT_PATH="$(readlink -f "$0" 2>/dev/null || echo "$0")"
|
|
case "$SCRIPT_PATH" in
|
|
/data/scripts/*)
|
|
log_warn "=================================================="
|
|
log_warn "Running from legacy path: $SCRIPT_PATH"
|
|
log_warn "Please update your docker-compose.upgrade.yml to use:"
|
|
log_warn " command: /data/.meshmonitor-internal/upgrade-watchdog.sh"
|
|
log_warn "See: https://github.com/Yeraze/meshmonitor/blob/main/docker-compose.upgrade.yml"
|
|
log_warn "=================================================="
|
|
;;
|
|
esac
|
|
|
|
# Initialize status
|
|
write_status "ready"
|
|
|
|
while true; do
|
|
# Check for Docker socket test request
|
|
if [ -f "$DOCKER_SOCKET_TEST_REQUEST" ]; then
|
|
log "Docker socket test request detected"
|
|
|
|
if [ -f "$DOCKER_SOCKET_TEST_SCRIPT" ]; then
|
|
# Make script executable and run it
|
|
chmod +x "$DOCKER_SOCKET_TEST_SCRIPT"
|
|
if sh "$DOCKER_SOCKET_TEST_SCRIPT"; then
|
|
log_success "Docker socket test completed"
|
|
else
|
|
log_warn "Docker socket test completed with warnings/errors"
|
|
fi
|
|
else
|
|
log_error "Docker socket test script not found: $DOCKER_SOCKET_TEST_SCRIPT"
|
|
echo "FAIL: Test script not found at $DOCKER_SOCKET_TEST_SCRIPT" > /data/.docker-socket-test
|
|
fi
|
|
|
|
# Clean up test request
|
|
rm -f "$DOCKER_SOCKET_TEST_REQUEST"
|
|
fi
|
|
|
|
# Check for upgrade trigger
|
|
if [ -f "$TRIGGER_FILE" ]; then
|
|
log "Upgrade trigger detected!"
|
|
|
|
if perform_upgrade; then
|
|
log_success "Upgrade process completed"
|
|
else
|
|
log_error "Upgrade process failed"
|
|
fi
|
|
|
|
# Clean up
|
|
rm -f "$TRIGGER_FILE"
|
|
fi
|
|
|
|
sleep "$CHECK_INTERVAL"
|
|
done
|
|
}
|
|
|
|
# Handle signals
|
|
trap 'log "Shutting down watchdog..."; exit 0' SIGTERM SIGINT
|
|
|
|
# Run main loop
|
|
main
|