170 lines
4.4 KiB
Bash
Executable File
170 lines
4.4 KiB
Bash
Executable File
#!/bin/bash
|
|
#===========================================
|
|
# 自动恢复机制脚本
|
|
# 监控关键进程，失败时自动重启
|
|
#===========================================
|
|
|
|
# 配置文件
|
|
CONFIG_FILE="/root/.openclaw/config/auto-recovery.conf"
|
|
LOG_FILE="/root/.openclaw/logs/auto-recovery.log"
|
|
ALERT_LOG="/root/.openclaw/logs/auto-recovery-alerts.log"
|
|
|
|
mkdir -p /root/.openclaw/logs
|
|
|
|
#===========================================
|
|
# 加载配置
|
|
#===========================================
|
|
load_config() {
|
|
if [ -f "$CONFIG_FILE" ]; then
|
|
source "$CONFIG_FILE"
|
|
else
|
|
# 默认配置
|
|
CHECK_INTERVAL=60
|
|
RESTART_DELAY=5
|
|
MAX_RESTART=3
|
|
fi
|
|
}
|
|
|
|
#===========================================
|
|
# 记录日志
|
|
#===========================================
|
|
log() {
|
|
local message=$1
|
|
local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
|
|
echo "[$timestamp] $message" >> "$LOG_FILE"
|
|
}
|
|
|
|
#===========================================
|
|
# 检查进程状态
|
|
#===========================================
|
|
check_process() {
|
|
local process_name=$1
|
|
local restart_cmd=$2
|
|
|
|
if pgrep -f "$process_name" > /dev/null; then
|
|
echo "✅ $process_name: 运行中"
|
|
return 0
|
|
else
|
|
echo "❌ $process_name: 未运行"
|
|
log "进程 $process_name 未运行，准备重启..."
|
|
|
|
# 尝试重启
|
|
local restart_count=1
|
|
while [ $restart_count -le $MAX_RESTART ]; do
|
|
log "尝试重启 $process_name (第 $restart_count 次)..."
|
|
|
|
eval "$restart_cmd" >> "$LOG_FILE" 2>&1
|
|
|
|
sleep $RESTART_DELAY
|
|
|
|
if pgrep -f "$process_name" > /dev/null; then
|
|
log "✅ $process_name 重启成功"
|
|
echo "✅ $process_name: 已重启"
|
|
return 0
|
|
fi
|
|
|
|
restart_count=$((restart_count + 1))
|
|
done
|
|
|
|
log "❌ $process_name 重启失败"
|
|
echo "❌ $process_name: 重启失败"
|
|
|
|
# 发送告警
|
|
echo "[$(date)] [CRITICAL] $process_name 自动恢复失败" >> "$ALERT_LOG"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
#===========================================
|
|
# 检查服务端口
|
|
#===========================================
|
|
check_port() {
|
|
local name=$1
|
|
local port=$2
|
|
|
|
if curl -s --noproxy 127.0.0.1 --max-time 5 http://127.0.0.1:$port/ > /dev/null 2>&1; then
|
|
echo "✅ $name 端口 $port: 正常"
|
|
return 0
|
|
else
|
|
echo "❌ $name 端口 $port: 无响应"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
#===========================================
|
|
# 自动恢复主逻辑
|
|
#===========================================
|
|
auto_recovery() {
|
|
echo "=== 自动恢复检查 $(date '+%Y-%m-%d %H:%M:%S') ==="
|
|
log "开始自动恢复检查..."
|
|
|
|
local failed=0
|
|
|
|
# 1. 检查 Gateway
|
|
echo ""
|
|
echo "检查 Gateway..."
|
|
if ! check_process "openclaw-gateway" "openclaw gateway restart"; then
|
|
((failed++))
|
|
fi
|
|
|
|
# 2. 检查端口
|
|
echo ""
|
|
echo "检查端口..."
|
|
check_port "Gateway" "18789"
|
|
|
|
# 3. 检查关键 OpenClaw 进程
|
|
echo ""
|
|
echo "检查 OpenClaw 进程..."
|
|
check_process "openclaw-gateway" "openclaw gateway restart"
|
|
|
|
# 总结
|
|
echo ""
|
|
if [ $failed -eq 0 ]; then
|
|
echo "✅ 自动恢复检查完成 - 所有服务正常"
|
|
log "自动恢复检查完成 - 所有服务正常"
|
|
else
|
|
echo "⚠️ 自动恢复检查完成 - $failed 项需要关注"
|
|
log "自动恢复检查完成 - $failed 项需要关注"
|
|
fi
|
|
}
|
|
|
|
#===========================================
|
|
# 持续监控模式
|
|
#===========================================
|
|
monitor_loop() {
|
|
load_config
|
|
|
|
echo "=== 自动恢复监控启动 ==="
|
|
echo "检查间隔: ${CHECK_INTERVAL}秒"
|
|
echo "最大重启次数: $MAX_RESTART"
|
|
log "自动恢复监控启动..."
|
|
|
|
while true; do
|
|
auto_recovery
|
|
sleep $CHECK_INTERVAL
|
|
done
|
|
}
|
|
|
|
#===========================================
|
|
# 命令行接口
|
|
#===========================================
|
|
case "${1:-check}" in
|
|
check)
|
|
load_config
|
|
auto_recovery
|
|
;;
|
|
monitor)
|
|
monitor_loop
|
|
;;
|
|
list)
|
|
load_config
|
|
echo "=== 自动恢复配置 ==="
|
|
echo "检查间隔: ${CHECK_INTERVAL}s"
|
|
echo "重启延迟: ${RESTART_DELAY}s"
|
|
echo "最大重启次数: $MAX_RESTART"
|
|
;;
|
|
*)
|
|
echo "用法: $0 {check|monitor|list}"
|
|
;;
|
|
esac
|