coop/deployment/health-check.sh

#!/bin/bash

# ==========================================
# CoopCredits Health Check Script
# Monitors all services in the infrastructure
# ==========================================

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Configuration
PLEX_IP="172.20.1.220"
OVERSEER_IP="172.20.1.225"
TAUTULLI_IP="172.20.1.255"

PLEX_PORT="32400"
OVERSEER_PORT="5055"
TAUTULLI_PORT="8181"

# Track overall health
HEALTHY=0
UNHEALTHY=0

print_header() {
    echo ""
    echo "🏥 CoopCredits Health Check"
    echo "=========================="
    echo "$(date)"
    echo ""
}

check_service() {
    local name=$1
    local url=$2
    local expected_code=${3:-200}
    
    echo -n "Checking $name... "
    
    if curl -sf -o /dev/null -w "%{http_code}" "$url" 2>/dev/null | grep -q "$expected_code"; then
        echo -e "${GREEN}✅ OK${NC}"
        ((HEALTHY++))
        return 0
    else
        echo -e "${RED}❌ FAIL${NC}"
        ((UNHEALTHY++))
        return 1
    fi
}

check_docker_service() {
    local name=$1
    local container=$2
    
    echo -n "Checking Docker container: $name... "
    
    if docker ps --format "{{.Names}}" | grep -q "^${container}$"; then
        local status=$(docker inspect --format='{{.State.Status}}' "$container" 2>/dev/null)
        if [ "$status" = "running" ]; then
            echo -e "${GREEN}✅ Running${NC}"
            ((HEALTHY++))
            return 0
        else
            echo -e "${RED}❌ Not running (status: $status)${NC}"
            ((UNHEALTHY++))
            return 1
        fi
    else
        echo -e "${RED}❌ Not found${NC}"
        ((UNHEALTHY++))
        return 1
    fi
}

check_network_connectivity() {
    local name=$1
    local ip=$2
    local port=$3
    
    echo -n "Network connectivity to $name ($ip:$port)... "
    
    if timeout 5 bash -c "</dev/tcp/$ip/$port" 2>/dev/null; then
        echo -e "${GREEN}✅ Reachable${NC}"
        ((HEALTHY++))
        return 0
    else
        echo -e "${RED}❌ Unreachable${NC}"
        ((UNHEALTHY++))
        return 1
    fi
}

check_disk_space() {
    echo ""
    echo "📊 Disk Space"
    echo "-------------"
    
    df -h / | awk 'NR==2 {printf "Root: %s used of %s (%s)\n", $3, $2, $5}'
    
    local usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
    if [ "$usage" -gt 90 ]; then
        echo -e "${RED}⚠️  Critical disk usage!${NC}"
        ((UNHEALTHY++))
    elif [ "$usage" -gt 80 ]; then
        echo -e "${YELLOW}⚠️  High disk usage${NC}"
    else
        echo -e "${GREEN}✅ Disk usage OK${NC}"
        ((HEALTHY++))
    fi
}

check_memory() {
    echo ""
    echo "🧠 Memory Usage"
    echo "--------------"
    
    if command -v free >/dev/null 2>&1; then
        free -h | grep "Mem:" | awk '{printf "Used: %s / %s\n", $3, $2}'
        
        local usage=$(free | grep "Mem:" | awk '{printf "%.0f", $3/$2 * 100}')
        if [ "$usage" -gt 90 ]; then
            echo -e "${RED}⚠️  Critical memory usage!${NC}"
            ((UNHEALTHY++))
        elif [ "$usage" -gt 80 ]; then
            echo -e "${YELLOW}⚠️  High memory usage${NC}"
        else
            echo -e "${GREEN}✅ Memory usage OK${NC}"
            ((HEALTHY++))
        fi
    else
        echo "Memory check not available"
    fi
}

check_ssl_certificate() {
    echo ""
    echo "🔒 SSL Certificate"
    echo "------------------"
    
    local domain="coop.hobokenchicken.com"
    local expiry=$(echo | openssl s_client -servername "$domain" -connect "$domain:443" 2>/dev/null | openssl x509 -noout -dates | grep notAfter | cut -d= -f2)
    
    if [ -n "$expiry" ]; then
        local expiry_epoch=$(date -d "$expiry" +%s 2>/dev/null || date -j -f "%b %d %H:%M:%S %Y %Z" "$expiry" +%s)
        local now_epoch=$(date +%s)
        local days_left=$(( (expiry_epoch - now_epoch) / 86400 ))
        
        echo "Certificate expires: $expiry ($days_left days)"
        
        if [ "$days_left" -lt 7 ]; then
            echo -e "${RED}⚠️  Certificate expires soon!${NC}"
            ((UNHEALTHY++))
        elif [ "$days_left" -lt 30 ]; then
            echo -e "${YELLOW}⚠️  Certificate expires in $days_left days${NC}"
        else
            echo -e "${GREEN}✅ Certificate valid${NC}"
            ((HEALTHY++))
        fi
    else
        echo -e "${YELLOW}⚠️  Could not check certificate${NC}"
    fi
}

run_checks() {
    print_header
    
    # Docker Services
    echo "🐳 Docker Services"
    echo "------------------"
    check_docker_service "Nginx" "coop-nginx"
    check_docker_service "Frontend" "coop-frontend"
    check_docker_service "Backend" "coop-backend"
    check_docker_service "PostgreSQL" "coop-postgres"
    check_docker_service "Redis" "coop-redis"
    
    # Network Connectivity
    echo ""
    echo "🌐 Network Connectivity"
    echo "-----------------------"
    check_network_connectivity "Plex" "$PLEX_IP" "$PLEX_PORT"
    check_network_connectivity "Tautulli" "$TAUTULLI_IP" "$TAUTULLI_PORT"
    check_network_connectivity "Overseer" "$OVERSEER_IP" "$OVERSEER_PORT"
    
    # External Services
    echo ""
    echo "🔗 External Services"
    echo "--------------------"
    check_service "CoopCredits Website" "https://coop.hobokenchicken.com/health"
    check_service "Backend API" "https://coop.hobokenchicken.com/api/health"
    
    # System Resources
    check_disk_space
    check_memory
    
    # SSL
    check_ssl_certificate
    
    # Summary
    echo ""
    echo "📋 Summary"
    echo "----------"
    echo -e "${GREEN}✅ $HEALTHY healthy${NC}"
    if [ $UNHEALTHY -gt 0 ]; then
        echo -e "${RED}❌ $UNHEALTHY unhealthy${NC}"
        exit 1
    else
        echo -e "${GREEN}All systems operational!${NC}"
        exit 0
    fi
}

# Run in watch mode if requested
if [ "$1" == "--watch" ]; then
    while true; do
        clear
        run_checks
        echo ""
        echo "Refreshing in 30 seconds... (Ctrl+C to stop)"
        sleep 30
    done
else
    run_checks
fi