meshtastic: add stack to repo, gitignore tiles, add scripts dir placeholder

This commit is contained in:
Paccoco
2026-05-05 12:15:05 -05:00
parent 462edb32b1
commit 0c87184ec4
6 changed files with 1824 additions and 0 deletions

View File

@@ -0,0 +1,570 @@
#!/bin/sh
# MeshMonitor Upgrade Watchdog
# Monitors for upgrade trigger file and performs Docker container upgrade
set -e
# Configuration
TRIGGER_FILE="${TRIGGER_FILE:-/data/.upgrade-trigger}"
STATUS_FILE="${STATUS_FILE:-/data/.upgrade-status}"
BACKUP_DIR="${BACKUP_DIR:-/data/backups}"
CHECK_INTERVAL="${CHECK_INTERVAL:-5}"
CONTAINER_NAME="${CONTAINER_NAME:-meshmonitor}"
IMAGE_NAME="${IMAGE_NAME:-ghcr.io/yeraze/meshmonitor}"
COMPOSE_PROJECT_DIR="${COMPOSE_PROJECT_DIR:-/compose}"
DOCKER_SOCKET_TEST_REQUEST="${DOCKER_SOCKET_TEST_REQUEST:-/data/.docker-socket-test-request}"
DOCKER_SOCKET_TEST_SCRIPT="${DOCKER_SOCKET_TEST_SCRIPT:-/data/.meshmonitor-internal/test-docker-socket.sh}"
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
log() {
echo "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}
log_success() {
echo "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC}$1"
}
log_warn() {
echo "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} ⚠️ $1"
}
log_error() {
echo "${RED}[$(date +'%Y-%m-%d %H:%M:%S')]${NC}$1"
}
# Write status to file for backend to read
write_status() {
echo "$1" > "$STATUS_FILE"
log "Status: $1"
}
# Create backup of data directory
create_backup() {
local backup_name="upgrade-backup-$(date +%Y%m%d_%H%M%S)"
local backup_path="$BACKUP_DIR/$backup_name"
log "Creating backup: $backup_path"
# Verify /data directory exists and is accessible
if [ ! -d "/data" ]; then
log_error "/data directory does not exist"
return 1
fi
# Create backup directory with error checking
if ! mkdir -p "$BACKUP_DIR" 2>/tmp/backup-error.log; then
log_error "Failed to create backup directory: $BACKUP_DIR"
if [ -f /tmp/backup-error.log ]; then
log_error "Error: $(cat /tmp/backup-error.log)"
rm -f /tmp/backup-error.log
fi
return 1
fi
# Verify backup directory was created
if [ ! -d "$BACKUP_DIR" ]; then
log_error "Backup directory does not exist after mkdir: $BACKUP_DIR"
return 1
fi
# Log directory info for debugging
log "Backup directory info: $(ls -ld "$BACKUP_DIR" 2>&1 || echo 'unable to stat')"
# Create backup (exclude backups directory itself)
# Capture stderr to a temp file for better error reporting
# Note: tar may report warnings about files disappearing (e.g., SQLite journal files)
# which can be safely ignored as long as the backup file is created successfully
tar -czf "$backup_path.tar.gz" -C /data --exclude='backups' --exclude='.upgrade-*' . 2>/tmp/tar-error.log
local tar_exit=$?
# Check if backup file was created and has content
if [ ! -f "$backup_path.tar.gz" ]; then
log_error "Backup file was not created: $backup_path.tar.gz"
# Show tar errors
if [ -f /tmp/tar-error.log ]; then
log_error "tar error output:"
while IFS= read -r line; do
log_error " $line"
done < /tmp/tar-error.log
fi
rm -f /tmp/tar-error.log
return 1
fi
local backup_size=$(stat -c%s "$backup_path.tar.gz" 2>/dev/null || echo "0")
# Check if backup is too small (likely corrupt)
if [ "$backup_size" -lt 100 ]; then
log_error "Backup file is suspiciously small (${backup_size} bytes)"
rm -f /tmp/tar-error.log
return 1
fi
# Exit code 0 = success, 1 = warnings (e.g., file changed during read)
# Both are acceptable as long as the backup file exists and has reasonable size
if [ $tar_exit -eq 0 ] || [ $tar_exit -eq 1 ]; then
if [ $tar_exit -eq 1 ] && [ -f /tmp/tar-error.log ]; then
# Log warnings but don't fail
log_warn "tar reported warnings (likely ephemeral files like SQLite journals):"
while IFS= read -r line; do
log_warn " $line"
done < /tmp/tar-error.log
fi
log_success "Backup created: $backup_path.tar.gz (${backup_size} bytes)"
rm -f /tmp/tar-error.log
echo "$backup_path.tar.gz"
return 0
else
log_error "Failed to create backup (tar exit code: $tar_exit)"
# Show the actual tar error
if [ -f /tmp/tar-error.log ]; then
log_error "tar error output:"
while IFS= read -r line; do
log_error " $line"
done < /tmp/tar-error.log
fi
rm -f /tmp/tar-error.log
return 1
fi
}
# Pull new Docker image
pull_image() {
local version="$1"
local image="${IMAGE_NAME}:${version}"
log "Pulling image: $image"
if docker pull "$image"; then
log_success "Image pulled: $image"
# Tag as latest if not already latest
if [ "$version" != "latest" ]; then
docker tag "$image" "${IMAGE_NAME}:latest"
log_success "Tagged as latest"
fi
return 0
else
log_error "Failed to pull image: $image"
return 1
fi
}
# Recreate container with new image
recreate_container() {
log "Recreating container: $CONTAINER_NAME"
# Resolve compose directory - try configured path, then known mount points
# Older sidecar configs used /data/compose, current uses /compose
# Accept both .yml and .yaml extensions (both are valid for Docker Compose)
local compose_dir=""
local compose_file=""
for try_dir in "$COMPOSE_PROJECT_DIR" "/compose" "/data/compose"; do
if [ -d "$try_dir" ]; then
if [ -f "$try_dir/docker-compose.yml" ]; then
compose_dir="$try_dir"
compose_file="docker-compose.yml"
break
elif [ -f "$try_dir/docker-compose.yaml" ]; then
compose_dir="$try_dir"
compose_file="docker-compose.yaml"
break
fi
fi
done
if [ -n "$compose_dir" ] && [ "$compose_dir" != "$COMPOSE_PROJECT_DIR" ]; then
if [ "$compose_dir" = "/data/compose" ]; then
log_warn "Falling back to legacy path /data/compose - please update COMPOSE_PROJECT_DIR to /compose"
else
log_warn "COMPOSE_PROJECT_DIR=$COMPOSE_PROJECT_DIR not found, falling back to $compose_dir"
fi
fi
if [ -z "$compose_dir" ]; then
log_error "No docker-compose.yml/yaml found at $COMPOSE_PROJECT_DIR, /compose, or /data/compose"
log_error "The upgrade sidecar requires Docker Compose files to recreate containers safely."
log_error "Mount your compose directory to /compose in the sidecar container."
return 1
fi
log "Using compose directory: $compose_dir"
# Verify docker compose is available
if ! docker compose version >/dev/null 2>&1; then
log_error "docker compose not available - required for container recreation"
return 1
fi
# Detect which compose files were originally used
local original_config_files=$(docker inspect --format='{{index .Config.Labels "com.docker.compose.project.config_files"}}' "$CONTAINER_NAME" 2>/dev/null || echo "")
local compose_files=""
if [ -n "$original_config_files" ]; then
log "Original compose files: $original_config_files"
for config_file in $(echo "$original_config_files" | tr ',' ' '); do
local filename=$(basename "$config_file")
if [ -f "$compose_dir/$filename" ]; then
compose_files="$compose_files -f $filename"
log " Using: $filename"
else
log_warn " Not found: $filename (skipping)"
fi
done
fi
# Fallback if no compose files detected from labels
if [ -z "$compose_files" ]; then
compose_files="-f $compose_file"
# Check for upgrade overlay with either extension
if [ -f "$compose_dir/docker-compose.upgrade.yml" ]; then
compose_files="$compose_files -f docker-compose.upgrade.yml"
elif [ -f "$compose_dir/docker-compose.upgrade.yaml" ]; then
compose_files="$compose_files -f docker-compose.upgrade.yaml"
fi
log "Using default compose files: $compose_files"
fi
# Detect project name from container labels
local detected_project=$(docker inspect --format='{{index .Config.Labels "com.docker.compose.project"}}' "$CONTAINER_NAME" 2>/dev/null || echo "")
local project_flag=""
if [ -n "$detected_project" ]; then
project_flag="-p $detected_project"
log "Using project name: $detected_project"
elif [ -n "$COMPOSE_PROJECT_NAME" ]; then
project_flag="-p $COMPOSE_PROJECT_NAME"
log "Using env project name: $COMPOSE_PROJECT_NAME"
fi
# Pull latest image
log "Pulling latest image..."
if docker pull "${IMAGE_NAME}:latest" 2>/dev/null; then
log_success "Image pulled: ${IMAGE_NAME}:latest"
fi
# Recreate via docker compose (handles ports, volumes, env, networks - everything)
cd "$compose_dir" || return 1
log "Running: docker compose $project_flag $compose_files up -d --force-recreate --no-deps $CONTAINER_NAME"
if docker compose $project_flag $compose_files up -d --force-recreate --no-deps "$CONTAINER_NAME"; then
# Log the port mappings on the new container for verification
local new_ports=$(docker port "$CONTAINER_NAME" 2>/dev/null)
if [ -n "$new_ports" ]; then
log "Port mappings on recreated container:"
echo "$new_ports" | while IFS= read -r line; do log " $line"; done
fi
log_success "Container recreated successfully via Docker Compose"
return 0
else
log_error "Failed to recreate container via Docker Compose"
return 1
fi
}
# Wait for container health check
wait_for_health() {
local max_wait=120
local elapsed=0
log "Waiting for container health check..."
# Get BASE_URL from container env vars, default to empty if not set
local base_url=$(docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$CONTAINER_NAME" 2>/dev/null | grep '^BASE_URL=' | cut -d'=' -f2 | tr -d '\r\n')
# Construct health endpoint URL with BASE_URL if present
local health_path="/api/health"
if [ -n "$base_url" ] && [ "$base_url" != "/" ]; then
health_path="${base_url}/api/health"
fi
while [ $elapsed -lt $max_wait ]; do
# Check if container is running
if ! docker ps --filter "name=$CONTAINER_NAME" --filter "status=running" | grep -q "$CONTAINER_NAME"; then
log_warn "Container not running yet..."
sleep 5
elapsed=$((elapsed + 5))
continue
fi
# Get container IP directly from Docker inspect - more reliable than DNS after recreation
# Try multiple networks as the container might be on different networks
local container_ip=""
container_ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_NAME" 2>/dev/null | head -n1)
# If no IP yet, wait for container to get network assigned
if [ -z "$container_ip" ]; then
log_warn "Container has no IP assigned yet..."
sleep 5
elapsed=$((elapsed + 5))
continue
fi
local health_url="http://${container_ip}:3001${health_path}"
# Only log URL on first attempt or if it changed
if [ "$elapsed" -eq 0 ] || [ -z "$last_health_url" ] || [ "$health_url" != "$last_health_url" ]; then
log "Health endpoint: $health_url"
last_health_url="$health_url"
fi
# Try to check health endpoint using container IP directly
# This is more reliable than container name DNS after recreation
if wget -q -O /dev/null --timeout=5 "$health_url" 2>/dev/null || \
curl -sf "$health_url" >/dev/null 2>&1; then
log_success "Health check passed"
return 0
fi
log "Waiting for health check... (${elapsed}s/${max_wait}s)"
sleep 5
elapsed=$((elapsed + 5))
done
log_error "Health check timeout after ${max_wait}s"
return 1
}
# Clean up old Docker images to free disk space
cleanup_old_images() {
log "Cleaning up old MeshMonitor images..."
# Get the current image ID being used by the container
local current_image_id=$(docker inspect --format='{{.Image}}' "$CONTAINER_NAME" 2>/dev/null | cut -d':' -f2 | cut -c1-12)
if [ -z "$current_image_id" ]; then
log_warn "Could not determine current image ID, skipping cleanup"
return 0
fi
log "Current image ID: $current_image_id"
# Find all MeshMonitor images (excluding the current one)
local old_images=$(docker images "${IMAGE_NAME}" --format '{{.ID}} {{.Tag}}' 2>/dev/null | while read id tag; do
# Get short ID for comparison
local short_id=$(echo "$id" | cut -c1-12)
if [ "$short_id" != "$current_image_id" ]; then
echo "$id"
fi
done)
if [ -z "$old_images" ]; then
log "No old images to clean up"
return 0
fi
# Count images to be removed
local image_count=$(echo "$old_images" | wc -l)
log "Found $image_count old image(s) to remove"
# Remove each old image
local removed=0
local failed=0
for image_id in $old_images; do
if docker rmi "$image_id" 2>/dev/null; then
removed=$((removed + 1))
log "Removed image: $image_id"
else
failed=$((failed + 1))
log_warn "Could not remove image: $image_id (may be in use)"
fi
done
if [ $removed -gt 0 ]; then
log_success "Cleaned up $removed old image(s)"
fi
if [ $failed -gt 0 ]; then
log_warn "$failed image(s) could not be removed"
fi
# Also clean up dangling images (untagged images from the build process)
local dangling=$(docker images -f "dangling=true" -q 2>/dev/null)
if [ -n "$dangling" ]; then
log "Removing dangling images..."
echo "$dangling" | xargs docker rmi 2>/dev/null || true
log_success "Dangling images cleaned up"
fi
return 0
}
# Perform upgrade
perform_upgrade() {
local trigger_data
local version
local backup_enabled
local upgrade_id
local backup_path
# Read trigger file
if [ ! -f "$TRIGGER_FILE" ]; then
log_error "Trigger file not found"
return 1
fi
trigger_data=$(cat "$TRIGGER_FILE")
version=$(echo "$trigger_data" | grep -o '"version"[ ]*:[ ]*"[^"]*"' | sed 's/.*:.*"\(.*\)"/\1/')
backup_enabled=$(echo "$trigger_data" | grep -o '"backup"[ ]*:[ ]*[^,}]*' | sed 's/.*:[ ]*//')
upgrade_id=$(echo "$trigger_data" | grep -o '"upgradeId"[ ]*:[ ]*"[^"]*"' | sed 's/.*:.*"\(.*\)"/\1/')
if [ -z "$version" ]; then
version="latest"
fi
log "=================================================="
log "Starting upgrade to version: $version"
log "Upgrade ID: $upgrade_id"
log "Backup enabled: $backup_enabled"
log "=================================================="
# Remove trigger file immediately to prevent re-triggering
rm -f "$TRIGGER_FILE"
# Step 1: Create backup
if [ "$backup_enabled" != "false" ]; then
write_status "backing_up"
if backup_path=$(create_backup); then
log_success "Backup completed: $backup_path"
else
write_status "failed"
log_error "Backup failed - aborting upgrade"
return 1
fi
else
log_warn "Backup disabled - skipping"
fi
# Step 2: Pull new image
write_status "downloading"
if ! pull_image "$version"; then
write_status "failed"
log_error "Image pull failed - aborting upgrade"
return 1
fi
# Step 3: Recreate container
write_status "restarting"
if ! recreate_container; then
write_status "failed"
log_error "Container recreation failed"
# Attempt rollback if backup exists
if [ -n "$backup_path" ] && [ -f "$backup_path" ]; then
log_warn "Attempting rollback..."
write_status "rolling_back"
# Rollback logic would go here
# For now, just log the error
log_error "Manual intervention required - backup available at: $backup_path"
fi
return 1
fi
# Step 4: Health check
write_status "health_check"
if ! wait_for_health; then
write_status "failed"
log_error "Health check failed - upgrade may have issues"
return 1
fi
# Step 5: Clean up old images to free disk space
write_status "cleanup"
cleanup_old_images
# Success!
write_status "complete"
log_success "=================================================="
log_success "Upgrade completed successfully!"
log_success "Version: $version"
log_success "Upgrade ID: $upgrade_id"
log_success "=================================================="
return 0
}
# Main loop
main() {
log "=================================================="
log "MeshMonitor Upgrade Watchdog Starting"
log "=================================================="
log "Container: $CONTAINER_NAME"
log "Image: $IMAGE_NAME"
log "Trigger file: $TRIGGER_FILE"
log "Check interval: ${CHECK_INTERVAL}s"
log "Compose project: $COMPOSE_PROJECT_DIR"
log "=================================================="
# Warn if running from the legacy script path
SCRIPT_PATH="$(readlink -f "$0" 2>/dev/null || echo "$0")"
case "$SCRIPT_PATH" in
/data/scripts/*)
log_warn "=================================================="
log_warn "Running from legacy path: $SCRIPT_PATH"
log_warn "Please update your docker-compose.upgrade.yml to use:"
log_warn " command: /data/.meshmonitor-internal/upgrade-watchdog.sh"
log_warn "See: https://github.com/Yeraze/meshmonitor/blob/main/docker-compose.upgrade.yml"
log_warn "=================================================="
;;
esac
# Initialize status
write_status "ready"
while true; do
# Check for Docker socket test request
if [ -f "$DOCKER_SOCKET_TEST_REQUEST" ]; then
log "Docker socket test request detected"
if [ -f "$DOCKER_SOCKET_TEST_SCRIPT" ]; then
# Make script executable and run it
chmod +x "$DOCKER_SOCKET_TEST_SCRIPT"
if sh "$DOCKER_SOCKET_TEST_SCRIPT"; then
log_success "Docker socket test completed"
else
log_warn "Docker socket test completed with warnings/errors"
fi
else
log_error "Docker socket test script not found: $DOCKER_SOCKET_TEST_SCRIPT"
echo "FAIL: Test script not found at $DOCKER_SOCKET_TEST_SCRIPT" > /data/.docker-socket-test
fi
# Clean up test request
rm -f "$DOCKER_SOCKET_TEST_REQUEST"
fi
# Check for upgrade trigger
if [ -f "$TRIGGER_FILE" ]; then
log "Upgrade trigger detected!"
if perform_upgrade; then
log_success "Upgrade process completed"
else
log_error "Upgrade process failed"
fi
# Clean up
rm -f "$TRIGGER_FILE"
fi
sleep "$CHECK_INTERVAL"
done
}
# Handle signals
trap 'log "Shutting down watchdog..."; exit 0' SIGTERM SIGINT
# Run main loop
main