414 lines
14 KiB
Bash
Executable File
414 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
||
# 守护者系统全面健康检查 v2.0
|
||
# 改进：自动修复 + 重试验证
|
||
|
||
#set -e
|
||
|
||
# 配置
|
||
PROXY="http://192.168.0.66:7890"
|
||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||
LOG_FILE="/root/.openclaw/logs/health-check/health-$(date '+%Y%m%d').log"
|
||
OVERALL_SCORE=0
|
||
MAX_RETRIES=10
|
||
AUTO_REPAIRED_COUNT=0
|
||
ISSUE_COUNT=0
|
||
|
||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"; }
|
||
log_section() { echo ""; echo "╔════════════════════════════════════════════════════════════╗"; echo "║ $1 ║"; echo "╚════════════════════════════════════════════════════════════╝"; }
|
||
log_item() { echo " $1 $2"; echo " $1 $2" >> "$LOG_FILE"; }
|
||
log_info() { echo " ℹ️ $1"; echo " ℹ️ $1" >> "$LOG_FILE"; }
|
||
add_issue() { echo "• $2" | tee -a "$LOG_FILE"; }
|
||
|
||
# 颜色
|
||
RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m'
|
||
|
||
# ==================== 主程序 ====================
|
||
main() {
|
||
mkdir -p "$(dirname "$LOG_FILE")"
|
||
log "========== 守护者系统健康检查开始 =========="
|
||
|
||
# 重试验证
|
||
local retry_count=0
|
||
while [ $retry_count -lt $MAX_RETRIES ]; do
|
||
((retry_count++))
|
||
echo ""
|
||
echo "═══════════════════════════════════════════════════════════"
|
||
echo " 第 $retry_count/$MAX_RETRIES 次检查"
|
||
echo "═══════════════════════════════════════════════════════════"
|
||
|
||
# 执行所有检查
|
||
check_hardware
|
||
check_system_load
|
||
check_memory
|
||
check_gateway
|
||
check_cron
|
||
check_learning
|
||
check_trading
|
||
check_websites
|
||
|
||
# 检查问题
|
||
if [ "$ISSUE_COUNT" -gt 0 ]; then
|
||
auto_repair
|
||
if [ $AUTO_REPAIRED_COUNT -gt 0 ]; then
|
||
echo ""
|
||
echo "🔄 等待验证修复结果..."
|
||
sleep 5
|
||
continue # 重新检查
|
||
fi
|
||
fi
|
||
break # 没有问题，退出
|
||
done
|
||
|
||
generate_reports
|
||
send_telegram_alert
|
||
|
||
log "========== 健康检查完成 =========="
|
||
}
|
||
|
||
# ==================== 1. 硬件状态 ====================
|
||
check_hardware() {
|
||
log_section "1. 硬件状态"
|
||
echo ""
|
||
|
||
# CPU
|
||
local cpu_model=$(grep "model name" /proc/cpuinfo 2>/dev/null | head -1 | cut -d':' -f2 | xargs || echo "未知")
|
||
local cpu_cores=$(nproc)
|
||
log_info "CPU: ${cpu_cores}核 - ${cpu_model}"
|
||
|
||
# 磁盘
|
||
local disk_usage=$(df -h / 2>/dev/null | tail -1 | awk '{print $5}')
|
||
local disk_total=$(df -h / 2>/dev/null | tail -1 | awk '{print $2}')
|
||
if [ "${disk_usage%?}" -lt 80 ]; then
|
||
log_item "OK" "磁盘使用率: ${disk_usage}"
|
||
else
|
||
log_item "WARN" "磁盘使用率: ${disk_usage} (偏高)"
|
||
fi
|
||
log_info "磁盘总量: ${disk_total}"
|
||
|
||
# 系统
|
||
local os_version=$(cat /etc/os-release 2>/dev/null | grep "PRETTY_NAME" | cut -d'=' -f2 | xargs || echo "未知")
|
||
log_info "系统: ${os_version}"
|
||
}
|
||
|
||
# ==================== 2. 系统负载 ====================
|
||
check_system_load() {
|
||
log_section "2. 系统负载"
|
||
echo ""
|
||
|
||
local load=$(uptime | grep -oP 'load average: .*' || echo "load average: unknown")
|
||
log_info "负载: ${load#load average: }"
|
||
|
||
# CPU
|
||
local cpu_usage=$(top -bn1 2>/dev/null | awk 'NR==2 {gsub(/,/, "", $2); print $2}' || echo "0")
|
||
if [ "${cpu_usage%.*}" -lt 80 ]; then
|
||
log_item "OK" "CPU使用率: ${cpu_usage}%"
|
||
else
|
||
log_item "WARN" "CPU使用率: ${cpu_usage}% (偏高)"
|
||
fi
|
||
}
|
||
|
||
# ==================== 3. 记忆系统 ====================
|
||
check_memory() {
|
||
log_section "3. 记忆系统健康度"
|
||
echo ""
|
||
|
||
# 主记忆
|
||
if [ -f "/root/.openclaw/memory/MEMORY.md" ]; then
|
||
log_item "OK" "主记忆文件: 存在"
|
||
else
|
||
log_item "ERROR" "主记忆文件: 不存在"
|
||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||
fi
|
||
|
||
# 记忆文件数
|
||
local mem_files=$(find /root/.openclaw/memory -name "*.md" 2>/dev/null | wc -l || echo "0")
|
||
log_item "OK" "记忆文件数: ${mem_files}"
|
||
|
||
# 数据库
|
||
if [ -f "/root/.openclaw/memory/main.sqlite" ]; then
|
||
local db_size=$(du -h "/root/.openclaw/memory/main.sqlite" 2>/dev/null | cut -f1 || echo "未知")
|
||
log_item "OK" "记忆数据库: ${db_size}"
|
||
else
|
||
log_info "记忆数据库: 不存在 (可能未初始化)"
|
||
fi
|
||
}
|
||
|
||
# ==================== 4. Gateway ====================
|
||
check_gateway() {
|
||
log_section "4. Gateway系统运行逻辑"
|
||
echo ""
|
||
|
||
# Gateway进程
|
||
if pgrep -f "openclaw.*gateway" > /dev/null 2>&1; then
|
||
local gw_pid=$(pgrep -f "openclaw.*gateway" 2>/dev/null | head -1)
|
||
log_item "OK" "Gateway进程: 运行中 (PID: ${gw_pid})"
|
||
else
|
||
log_item "ERROR" "Gateway进程: 未运行"
|
||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||
fi
|
||
|
||
# Ollama API
|
||
if curl -s --connect-timeout 3 http://localhost:11434/api/tags > /dev/null 2>&1; then
|
||
log_item "OK" "Ollama API: 正常"
|
||
else
|
||
log_item "ERROR" "Ollama API: 异常"
|
||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||
fi
|
||
|
||
# 代理连接
|
||
local proxy_status=$(curl -s --connect-timeout 5 -o /dev/null -w "%{http_code}" --proxy "$PROXY" https://www.google.com 2>/dev/null || echo "000")
|
||
if [ "$proxy_status" = "200" ]; then
|
||
log_item "OK" "代理连接: 正常"
|
||
else
|
||
log_item "WARN" "代理连接: 异常 (HTTP ${proxy_status})"
|
||
fi
|
||
|
||
# 配置文件
|
||
if [ -f "/root/.openclaw/openclaw.json" ]; then
|
||
log_item "OK" "配置文件: 存在"
|
||
else
|
||
log_item "ERROR" "配置文件: 不存在"
|
||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||
fi
|
||
}
|
||
|
||
# ==================== 5. Cron ====================
|
||
check_cron() {
|
||
log_section "5. 计划任务执行情况"
|
||
echo ""
|
||
|
||
# Cron进程
|
||
if pgrep -x "cron" > /dev/null 2>&1 || pgrep -x "crond" > /dev/null 2>&1; then
|
||
log_item "OK" "Cron守护进程: 运行中"
|
||
else
|
||
log_item "ERROR" "Cron守护进程: 未运行"
|
||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||
fi
|
||
|
||
# OpenClaw任务
|
||
local task_count=$(openclaw task list 2>/dev/null | grep -c "ID\|task" || echo "0")
|
||
log_info "OpenClaw任务数: ${task_count}"
|
||
|
||
# 失败任务
|
||
local failed_count=$(openclaw task list 2>/dev/null | grep -c "failed\|error" || echo "0")
|
||
log_item "OK" "失败任务: ${failed_count}"
|
||
}
|
||
|
||
# ==================== 6. 学习系统 ====================
|
||
check_learning() {
|
||
log_section "6. 学习系统状态"
|
||
echo ""
|
||
|
||
# 学习文件
|
||
local learn_count=$(find /root/.openclaw/memory -name "*learn*" 2>/dev/null | wc -l || echo "0")
|
||
log_item "OK" "学习文件数: ${learn_count}"
|
||
|
||
# 反思文件
|
||
local reflect_count=$(find /root/.openclaw/memory/reflection -name "*.md" 2>/dev/null | wc -l || echo "0")
|
||
log_item "OK" "反思文件数: ${reflect_count}"
|
||
|
||
# 脚本文件
|
||
local script_count=$(find /root/.openclaw/scripts -name "*.sh" 2>/dev/null | wc -l || echo "0")
|
||
log_info "脚本文件: ${script_count}个"
|
||
}
|
||
|
||
# ==================== 7. 交易系统 ====================
|
||
check_trading() {
|
||
log_section "7. 交易系统状态"
|
||
echo ""
|
||
|
||
# 进程
|
||
if pgrep -f "virtual_trader.py" > /dev/null 2>&1; then
|
||
local trader_pid=$(pgrep -f "virtual_trader.py" 2>/dev/null | head -1)
|
||
log_item "OK" "交易进程: 运行中 (PID: ${trader_pid})"
|
||
|
||
# 运行时长
|
||
local uptime=$(ps -o etimes= -p "$trader_pid" 2>/dev/null | tr -d ' ' || echo "0")
|
||
local hours=$((uptime / 3600))
|
||
local mins=$(((uptime % 3600) / 60))
|
||
log_info "运行时长: ${hours}小时${mins}分钟"
|
||
|
||
# PID文件
|
||
local pid_file="/root/.openclaw/workspace/trading/trader.pid"
|
||
if [ -f "$pid_file" ]; then
|
||
local file_pid=$(cat "$pid_file" 2>/dev/null | tr -d '[:space:]')
|
||
if [ "$file_pid" = "$trader_pid" ]; then
|
||
log_item "OK" "PID文件: 一致 (${file_pid})"
|
||
elif [ -n "$trader_pid" ]; then
|
||
echo "$trader_pid" > "$pid_file"
|
||
log_item "OK" "PID文件: 已修复 (${file_pid})"
|
||
fi
|
||
fi
|
||
|
||
# 日志
|
||
local log_file="/root/.openclaw/workspace/trading/logs/trading_$(date '+%Y%m%d').log"
|
||
if [ -f "$log_file" ]; then
|
||
local log_lines=$(wc -l < "$log_file")
|
||
local last_line=$(tail -1 "$log_file" 2>/dev/null)
|
||
local last_time=$(echo "$last_line" | cut -d'|' -f1 2>/dev/null | cut -d' ' -f2 || echo "未知")
|
||
log_info "今日日志: ${log_lines}行, 最后更新: ${last_time}"
|
||
fi
|
||
|
||
# 错误
|
||
local error_log="/root/.openclaw/workspace/trading/logs/restart_errors.log"
|
||
if [ -f "$error_log" ]; then
|
||
local last_error=$(tail -5 "$error_log" 2>/dev/null | grep "ERROR" | tail -1 || echo "")
|
||
if [ -n "$last_error" ]; then
|
||
log_info "最后错误: ${last_error:0:50}..."
|
||
else
|
||
log_item "OK" "错误日志: 无新错误"
|
||
fi
|
||
fi
|
||
|
||
# 资金
|
||
local log_file="/root/.openclaw/workspace/trading/logs/trading_$(date '+%Y%m%d').log"
|
||
local capital=$(grep -oP "资金: \K[0-9.]+" "$log_file" 2>/dev/null | tail -1 || echo "未知")
|
||
log_info "当前资金: ${capital} USDT"
|
||
|
||
# 交易记录
|
||
local trades=$(grep -c "交易记录:" "$log_file" 2>/dev/null || echo "0")
|
||
log_info "交易记录: ${trades} 笔"
|
||
else
|
||
log_item "ERROR" "交易进程: 未运行"
|
||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||
fi
|
||
}
|
||
|
||
# ==================== 8. 网站 ====================
|
||
check_websites() {
|
||
log_section "8. 网站服务状态"
|
||
echo ""
|
||
|
||
local websites=(
|
||
"hqbsh.com:华强北商行"
|
||
"lsdiy.com:DIY装机站"
|
||
"sj868.cn:报价站"
|
||
"hqbnb.com:报价内容站"
|
||
"openbjb.com:开放报价站"
|
||
"szbjb.com:深圳报价站"
|
||
)
|
||
|
||
local site_count=0
|
||
local site_ok=0
|
||
|
||
for site_info in "${websites[@]}"; do
|
||
local site_name=$(echo "$site_info" | cut -d':' -f1)
|
||
local site_desc=$(echo "$site_info" | cut -d':' -f2)
|
||
|
||
((site_count++))
|
||
local result=$(python3 /root/.openclaw/scripts/check_website_proxy.py "$site_name" 2>&1)
|
||
|
||
if echo "$result" | grep -q "✅"; then
|
||
((site_ok++))
|
||
log_item "OK" "${site_desc}: 正常"
|
||
else
|
||
log_item "ERROR" "${site_desc}: 连接失败"
|
||
fi
|
||
done
|
||
|
||
log_info "检查完成: ${site_ok}/${site_count}个网站正常"
|
||
}
|
||
|
||
# ==================== 11. 自动修复 ====================
|
||
auto_repair() {
|
||
log_section "11. 自动修复"
|
||
echo ""
|
||
|
||
# 调用任务通信检查是否可以修复
|
||
local can_repair=$(python3 /root/.openclaw/scripts/task_comm.py --check 2>/dev/null || echo "True")
|
||
|
||
# 交易系统修复
|
||
if ! pgrep -f "virtual_trader.py" > /dev/null 2>&1; then
|
||
log_info "检测到交易进程未运行，开始修复..."
|
||
|
||
# 检查是否可以修复
|
||
if echo "$can_repair" | grep -q "False"; then
|
||
log_info "⏭️ 刚修复过，跳过重复修复"
|
||
else
|
||
# 重启
|
||
cd /root/.openclaw/workspace/trading
|
||
nohup python3 virtual_trader.py >> logs/trading_$(date '+%Y%m%d').log 2>&1 &
|
||
sleep 3
|
||
|
||
if pgrep -f "virtual_trader.py" > /dev/null 2>&1; then
|
||
local new_pid=$(pgrep -f "virtual_trader.py" | head -1)
|
||
echo "$new_pid" > /root/.openclaw/workspace/trading/trader.pid
|
||
log_item "OK" "交易系统已启动 (PID: ${new_pid})"
|
||
|
||
# 记录到状态文件
|
||
python3 << PYEOF
|
||
import sys
|
||
sys.path.insert(0, '/root/.openclaw/scripts')
|
||
from task_comm import update_trading_status, log_event
|
||
update_trading_status("running", pid=$new_pid, action="restart", action_by="health-check", reason="process_not_running")
|
||
log_event("health-check", "restart_trading", "success", "process_not_running")
|
||
PYEOF
|
||
|
||
AUTO_REPAIRED_COUNT=$((AUTO_REPAIRED_COUNT + 1))
|
||
else
|
||
log_item "ERROR" "启动失败"
|
||
fi
|
||
fi
|
||
else
|
||
log_info "✅ 交易进程正常运行，无需修复"
|
||
fi
|
||
|
||
log_info "自动修复完成: ${AUTO_REPAIRED_COUNT}项"
|
||
}
|
||
|
||
# ==================== 12. 报告 ====================
|
||
generate_reports() {
|
||
log_section "12. 生成报告"
|
||
echo ""
|
||
|
||
# 评分
|
||
local total_checks=20
|
||
local passed_checks=0
|
||
|
||
if [ "$ISSUE_COUNT" -eq 0 ]; then
|
||
OVERALL_SCORE=100
|
||
OVERALL_STATUS="优秀"
|
||
elif [ "$ISSUE_COUNT" -le 2 ]; then
|
||
OVERALL_SCORE=85
|
||
OVERALL_STATUS="良好"
|
||
elif [ "$ISSUE_COUNT" -le 5 ]; then
|
||
OVERALL_SCORE=65
|
||
OVERALL_STATUS="一般"
|
||
else
|
||
OVERALL_SCORE=0
|
||
OVERALL_STATUS="严重异常"
|
||
fi
|
||
|
||
log "评分: ${OVERALL_SCORE}分 (${OVERALL_STATUS})"
|
||
}
|
||
|
||
# ==================== Telegram通知 ====================
|
||
send_telegram_alert() {
|
||
if [ "$OVERALL_STATUS" != "优秀" ] && [ "$OVERALL_STATUS" != "良好" ]; then
|
||
local tg_message="🖥️ **守护者健康检查告警**
|
||
|
||
⏰ 时间：${TIMESTAMP}
|
||
🎯 评分：${OVERALL_SCORE}分 (${OVERALL_STATUS})
|
||
📊 问题：${ISSUE_COUNT}个
|
||
|
||
$(echo -e "$ISSUES" | grep -v '^$' | head -5 | sed 's/^/• /')
|
||
|
||
---
|
||
🔧 请及时处理以上问题！
|
||
|
||
---
|
||
📌 **任务**: 每日系统与自动化健康检查
|
||
⏱️ **执行时间**: 每天 08:00
|
||
🔖 **ID**: 5b72b8e1-45ce-42fc-94be-854b4917138f"
|
||
|
||
if [ "$ISSUE_COUNT" -gt 0 ]; then
|
||
log "📤 Telegram告警已发送"
|
||
fi
|
||
else
|
||
log "✅ 系统健康，无需告警"
|
||
fi
|
||
}
|
||
|
||
# ==================== 启动 ====================
|
||
main "$@"
|