每日备份 2026-06-10 12:57:13
This commit is contained in:
root-backup committed
1 parent
fda2ccb8c2
commit
3d2e638c6a
244 files changed
+58890
-3
No files matched your search
+413
@@ -0,0 +1,413 @@
|
||||
#!/bin/bash
|
||||
# 守护者系统全面健康检查 v2.0
|
||||
# 改进：自动修复 + 重试验证
|
||||
|
||||
#set -e
|
||||
|
||||
# 配置
|
||||
PROXY="http://192.168.0.66:7890"
|
||||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
LOG_FILE="/root/.openclaw/logs/health-check/health-$(date '+%Y%m%d').log"
|
||||
OVERALL_SCORE=0
|
||||
MAX_RETRIES=10
|
||||
AUTO_REPAIRED_COUNT=0
|
||||
ISSUE_COUNT=0
|
||||
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"; }
|
||||
log_section() { echo ""; echo "╔════════════════════════════════════════════════════════════╗"; echo "║ $1 ║"; echo "╚════════════════════════════════════════════════════════════╝"; }
|
||||
log_item() { echo " $1 $2"; echo " $1 $2" >> "$LOG_FILE"; }
|
||||
log_info() { echo " ℹ️ $1"; echo " ℹ️ $1" >> "$LOG_FILE"; }
|
||||
add_issue() { echo "• $2" | tee -a "$LOG_FILE"; }
|
||||
|
||||
# 颜色
|
||||
RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m'
|
||||
|
||||
# ==================== 主程序 ====================
|
||||
main() {
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
log "========== 守护者系统健康检查开始 =========="
|
||||
|
||||
# 重试验证
|
||||
local retry_count=0
|
||||
while [ $retry_count -lt $MAX_RETRIES ]; do
|
||||
((retry_count++))
|
||||
echo ""
|
||||
echo "═══════════════════════════════════════════════════════════"
|
||||
echo " 第 $retry_count/$MAX_RETRIES 次检查"
|
||||
echo "═══════════════════════════════════════════════════════════"
|
||||
|
||||
# 执行所有检查
|
||||
check_hardware
|
||||
check_system_load
|
||||
check_memory
|
||||
check_gateway
|
||||
check_cron
|
||||
check_learning
|
||||
check_trading
|
||||
check_websites
|
||||
|
||||
# 检查问题
|
||||
if [ "$ISSUE_COUNT" -gt 0 ]; then
|
||||
auto_repair
|
||||
if [ $AUTO_REPAIRED_COUNT -gt 0 ]; then
|
||||
echo ""
|
||||
echo "🔄 等待验证修复结果..."
|
||||
sleep 5
|
||||
continue # 重新检查
|
||||
fi
|
||||
fi
|
||||
break # 没有问题，退出
|
||||
done
|
||||
|
||||
generate_reports
|
||||
send_telegram_alert
|
||||
|
||||
log "========== 健康检查完成 =========="
|
||||
}
|
||||
|
||||
# ==================== 1. 硬件状态 ====================
|
||||
check_hardware() {
|
||||
log_section "1. 硬件状态"
|
||||
echo ""
|
||||
|
||||
# CPU
|
||||
local cpu_model=$(grep "model name" /proc/cpuinfo 2>/dev/null | head -1 | cut -d':' -f2 | xargs || echo "未知")
|
||||
local cpu_cores=$(nproc)
|
||||
log_info "CPU: ${cpu_cores}核 - ${cpu_model}"
|
||||
|
||||
# 磁盘
|
||||
local disk_usage=$(df -h / 2>/dev/null | tail -1 | awk '{print $5}')
|
||||
local disk_total=$(df -h / 2>/dev/null | tail -1 | awk '{print $2}')
|
||||
if [ "${disk_usage%?}" -lt 80 ]; then
|
||||
log_item "OK" "磁盘使用率: ${disk_usage}"
|
||||
else
|
||||
log_item "WARN" "磁盘使用率: ${disk_usage} (偏高)"
|
||||
fi
|
||||
log_info "磁盘总量: ${disk_total}"
|
||||
|
||||
# 系统
|
||||
local os_version=$(cat /etc/os-release 2>/dev/null | grep "PRETTY_NAME" | cut -d'=' -f2 | xargs || echo "未知")
|
||||
log_info "系统: ${os_version}"
|
||||
}
|
||||
|
||||
# ==================== 2. 系统负载 ====================
|
||||
check_system_load() {
|
||||
log_section "2. 系统负载"
|
||||
echo ""
|
||||
|
||||
local load=$(uptime | grep -oP 'load average: .*' || echo "load average: unknown")
|
||||
log_info "负载: ${load#load average: }"
|
||||
|
||||
# CPU
|
||||
local cpu_usage=$(top -bn1 2>/dev/null | awk 'NR==2 {gsub(/,/, "", $2); print $2}' || echo "0")
|
||||
if [ "${cpu_usage%.*}" -lt 80 ]; then
|
||||
log_item "OK" "CPU使用率: ${cpu_usage}%"
|
||||
else
|
||||
log_item "WARN" "CPU使用率: ${cpu_usage}% (偏高)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==================== 3. 记忆系统 ====================
|
||||
check_memory() {
|
||||
log_section "3. 记忆系统健康度"
|
||||
echo ""
|
||||
|
||||
# 主记忆
|
||||
if [ -f "/root/.openclaw/memory/MEMORY.md" ]; then
|
||||
log_item "OK" "主记忆文件: 存在"
|
||||
else
|
||||
log_item "ERROR" "主记忆文件: 不存在"
|
||||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||||
fi
|
||||
|
||||
# 记忆文件数
|
||||
local mem_files=$(find /root/.openclaw/memory -name "*.md" 2>/dev/null | wc -l || echo "0")
|
||||
log_item "OK" "记忆文件数: ${mem_files}"
|
||||
|
||||
# 数据库
|
||||
if [ -f "/root/.openclaw/memory/main.sqlite" ]; then
|
||||
local db_size=$(du -h "/root/.openclaw/memory/main.sqlite" 2>/dev/null | cut -f1 || echo "未知")
|
||||
log_item "OK" "记忆数据库: ${db_size}"
|
||||
else
|
||||
log_info "记忆数据库: 不存在 (可能未初始化)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==================== 4. Gateway ====================
|
||||
check_gateway() {
|
||||
log_section "4. Gateway系统运行逻辑"
|
||||
echo ""
|
||||
|
||||
# Gateway进程
|
||||
if pgrep -f "openclaw.*gateway" > /dev/null 2>&1; then
|
||||
local gw_pid=$(pgrep -f "openclaw.*gateway" 2>/dev/null | head -1)
|
||||
log_item "OK" "Gateway进程: 运行中 (PID: ${gw_pid})"
|
||||
else
|
||||
log_item "ERROR" "Gateway进程: 未运行"
|
||||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||||
fi
|
||||
|
||||
# Ollama API
|
||||
if curl -s --connect-timeout 3 http://localhost:11434/api/tags > /dev/null 2>&1; then
|
||||
log_item "OK" "Ollama API: 正常"
|
||||
else
|
||||
log_item "ERROR" "Ollama API: 异常"
|
||||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||||
fi
|
||||
|
||||
# 代理连接
|
||||
local proxy_status=$(curl -s --connect-timeout 5 -o /dev/null -w "%{http_code}" --proxy "$PROXY" https://www.google.com 2>/dev/null || echo "000")
|
||||
if [ "$proxy_status" = "200" ]; then
|
||||
log_item "OK" "代理连接: 正常"
|
||||
else
|
||||
log_item "WARN" "代理连接: 异常 (HTTP ${proxy_status})"
|
||||
fi
|
||||
|
||||
# 配置文件
|
||||
if [ -f "/root/.openclaw/openclaw.json" ]; then
|
||||
log_item "OK" "配置文件: 存在"
|
||||
else
|
||||
log_item "ERROR" "配置文件: 不存在"
|
||||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# ==================== 5. Cron ====================
|
||||
check_cron() {
|
||||
log_section "5. 计划任务执行情况"
|
||||
echo ""
|
||||
|
||||
# Cron进程
|
||||
if pgrep -x "cron" > /dev/null 2>&1 || pgrep -x "crond" > /dev/null 2>&1; then
|
||||
log_item "OK" "Cron守护进程: 运行中"
|
||||
else
|
||||
log_item "ERROR" "Cron守护进程: 未运行"
|
||||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||||
fi
|
||||
|
||||
# OpenClaw任务
|
||||
local task_count=$(openclaw task list 2>/dev/null | grep -c "ID\|task" || echo "0")
|
||||
log_info "OpenClaw任务数: ${task_count}"
|
||||
|
||||
# 失败任务
|
||||
local failed_count=$(openclaw task list 2>/dev/null | grep -c "failed\|error" || echo "0")
|
||||
log_item "OK" "失败任务: ${failed_count}"
|
||||
}
|
||||
|
||||
# ==================== 6. 学习系统 ====================
|
||||
check_learning() {
|
||||
log_section "6. 学习系统状态"
|
||||
echo ""
|
||||
|
||||
# 学习文件
|
||||
local learn_count=$(find /root/.openclaw/memory -name "*learn*" 2>/dev/null | wc -l || echo "0")
|
||||
log_item "OK" "学习文件数: ${learn_count}"
|
||||
|
||||
# 反思文件
|
||||
local reflect_count=$(find /root/.openclaw/memory/reflection -name "*.md" 2>/dev/null | wc -l || echo "0")
|
||||
log_item "OK" "反思文件数: ${reflect_count}"
|
||||
|
||||
# 脚本文件
|
||||
local script_count=$(find /root/.openclaw/scripts -name "*.sh" 2>/dev/null | wc -l || echo "0")
|
||||
log_info "脚本文件: ${script_count}个"
|
||||
}
|
||||
|
||||
# ==================== 7. 交易系统 ====================
|
||||
check_trading() {
|
||||
log_section "7. 交易系统状态"
|
||||
echo ""
|
||||
|
||||
# 进程
|
||||
if pgrep -f "virtual_trader.py" > /dev/null 2>&1; then
|
||||
local trader_pid=$(pgrep -f "virtual_trader.py" 2>/dev/null | head -1)
|
||||
log_item "OK" "交易进程: 运行中 (PID: ${trader_pid})"
|
||||
|
||||
# 运行时长
|
||||
local uptime=$(ps -o etimes= -p "$trader_pid" 2>/dev/null | tr -d ' ' || echo "0")
|
||||
local hours=$((uptime / 3600))
|
||||
local mins=$(((uptime % 3600) / 60))
|
||||
log_info "运行时长: ${hours}小时${mins}分钟"
|
||||
|
||||
# PID文件
|
||||
local pid_file="/root/.openclaw/workspace/trading/trader.pid"
|
||||
if [ -f "$pid_file" ]; then
|
||||
local file_pid=$(cat "$pid_file" 2>/dev/null | tr -d '[:space:]')
|
||||
if [ "$file_pid" = "$trader_pid" ]; then
|
||||
log_item "OK" "PID文件: 一致 (${file_pid})"
|
||||
elif [ -n "$trader_pid" ]; then
|
||||
echo "$trader_pid" > "$pid_file"
|
||||
log_item "OK" "PID文件: 已修复 (${file_pid})"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 日志
|
||||
local log_file="/root/.openclaw/workspace/trading/logs/trading_$(date '+%Y%m%d').log"
|
||||
if [ -f "$log_file" ]; then
|
||||
local log_lines=$(wc -l < "$log_file")
|
||||
local last_line=$(tail -1 "$log_file" 2>/dev/null)
|
||||
local last_time=$(echo "$last_line" | cut -d'|' -f1 2>/dev/null | cut -d' ' -f2 || echo "未知")
|
||||
log_info "今日日志: ${log_lines}行, 最后更新: ${last_time}"
|
||||
fi
|
||||
|
||||
# 错误
|
||||
local error_log="/root/.openclaw/workspace/trading/logs/restart_errors.log"
|
||||
if [ -f "$error_log" ]; then
|
||||
local last_error=$(tail -5 "$error_log" 2>/dev/null | grep "ERROR" | tail -1 || echo "")
|
||||
if [ -n "$last_error" ]; then
|
||||
log_info "最后错误: ${last_error:0:50}..."
|
||||
else
|
||||
log_item "OK" "错误日志: 无新错误"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 资金
|
||||
local log_file="/root/.openclaw/workspace/trading/logs/trading_$(date '+%Y%m%d').log"
|
||||
local capital=$(grep -oP "资金: \K[0-9.]+" "$log_file" 2>/dev/null | tail -1 || echo "未知")
|
||||
log_info "当前资金: ${capital} USDT"
|
||||
|
||||
# 交易记录
|
||||
local trades=$(grep -c "交易记录:" "$log_file" 2>/dev/null || echo "0")
|
||||
log_info "交易记录: ${trades} 笔"
|
||||
else
|
||||
log_item "ERROR" "交易进程: 未运行"
|
||||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# ==================== 8. 网站 ====================
|
||||
check_websites() {
|
||||
log_section "8. 网站服务状态"
|
||||
echo ""
|
||||
|
||||
local websites=(
|
||||
"hqbsh.com:华强北商行"
|
||||
"lsdiy.com:DIY装机站"
|
||||
"sj868.cn:报价站"
|
||||
"hqbnb.com:报价内容站"
|
||||
"openbjb.com:开放报价站"
|
||||
"szbjb.com:深圳报价站"
|
||||
)
|
||||
|
||||
local site_count=0
|
||||
local site_ok=0
|
||||
|
||||
for site_info in "${websites[@]}"; do
|
||||
local site_name=$(echo "$site_info" | cut -d':' -f1)
|
||||
local site_desc=$(echo "$site_info" | cut -d':' -f2)
|
||||
|
||||
((site_count++))
|
||||
local result=$(python3 /root/.openclaw/scripts/check_website_proxy.py "$site_name" 2>&1)
|
||||
|
||||
if echo "$result" | grep -q "✅"; then
|
||||
((site_ok++))
|
||||
log_item "OK" "${site_desc}: 正常"
|
||||
else
|
||||
log_item "ERROR" "${site_desc}: 连接失败"
|
||||
fi
|
||||
done
|
||||
|
||||
log_info "检查完成: ${site_ok}/${site_count}个网站正常"
|
||||
}
|
||||
|
||||
# ==================== 11. 自动修复 ====================
|
||||
auto_repair() {
|
||||
log_section "11. 自动修复"
|
||||
echo ""
|
||||
|
||||
# 调用任务通信检查是否可以修复
|
||||
local can_repair=$(python3 /root/.openclaw/scripts/task_comm.py --check 2>/dev/null || echo "True")
|
||||
|
||||
# 交易系统修复
|
||||
if ! pgrep -f "virtual_trader.py" > /dev/null 2>&1; then
|
||||
log_info "检测到交易进程未运行，开始修复..."
|
||||
|
||||
# 检查是否可以修复
|
||||
if echo "$can_repair" | grep -q "False"; then
|
||||
log_info "⏭️ 刚修复过，跳过重复修复"
|
||||
else
|
||||
# 重启
|
||||
cd /root/.openclaw/workspace/trading
|
||||
nohup python3 virtual_trader.py >> logs/trading_$(date '+%Y%m%d').log 2>&1 &
|
||||
sleep 3
|
||||
|
||||
if pgrep -f "virtual_trader.py" > /dev/null 2>&1; then
|
||||
local new_pid=$(pgrep -f "virtual_trader.py" | head -1)
|
||||
echo "$new_pid" > /root/.openclaw/workspace/trading/trader.pid
|
||||
log_item "OK" "交易系统已启动 (PID: ${new_pid})"
|
||||
|
||||
# 记录到状态文件
|
||||
python3 << PYEOF
|
||||
import sys
|
||||
sys.path.insert(0, '/root/.openclaw/scripts')
|
||||
from task_comm import update_trading_status, log_event
|
||||
update_trading_status("running", pid=$new_pid, action="restart", action_by="health-check", reason="process_not_running")
|
||||
log_event("health-check", "restart_trading", "success", "process_not_running")
|
||||
PYEOF
|
||||
|
||||
AUTO_REPAIRED_COUNT=$((AUTO_REPAIRED_COUNT + 1))
|
||||
else
|
||||
log_item "ERROR" "启动失败"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log_info "✅ 交易进程正常运行，无需修复"
|
||||
fi
|
||||
|
||||
log_info "自动修复完成: ${AUTO_REPAIRED_COUNT}项"
|
||||
}
|
||||
|
||||
# ==================== 12. 报告 ====================
|
||||
generate_reports() {
|
||||
log_section "12. 生成报告"
|
||||
echo ""
|
||||
|
||||
# 评分
|
||||
local total_checks=20
|
||||
local passed_checks=0
|
||||
|
||||
if [ "$ISSUE_COUNT" -eq 0 ]; then
|
||||
OVERALL_SCORE=100
|
||||
OVERALL_STATUS="优秀"
|
||||
elif [ "$ISSUE_COUNT" -le 2 ]; then
|
||||
OVERALL_SCORE=85
|
||||
OVERALL_STATUS="良好"
|
||||
elif [ "$ISSUE_COUNT" -le 5 ]; then
|
||||
OVERALL_SCORE=65
|
||||
OVERALL_STATUS="一般"
|
||||
else
|
||||
OVERALL_SCORE=0
|
||||
OVERALL_STATUS="严重异常"
|
||||
fi
|
||||
|
||||
log "评分: ${OVERALL_SCORE}分 (${OVERALL_STATUS})"
|
||||
}
|
||||
|
||||
# ==================== Telegram通知 ====================
|
||||
send_telegram_alert() {
|
||||
if [ "$OVERALL_STATUS" != "优秀" ] && [ "$OVERALL_STATUS" != "良好" ]; then
|
||||
local tg_message="🖥️ **守护者健康检查告警**
|
||||
|
||||
⏰ 时间：${TIMESTAMP}
|
||||
🎯 评分：${OVERALL_SCORE}分 (${OVERALL_STATUS})
|
||||
📊 问题：${ISSUE_COUNT}个
|
||||
|
||||
$(echo -e "$ISSUES" | grep -v '^$' | head -5 | sed 's/^/• /')
|
||||
|
||||
---
|
||||
🔧 请及时处理以上问题！
|
||||
|
||||
---
|
||||
📌 **任务**: 每日系统与自动化健康检查
|
||||
⏱️ **执行时间**: 每天 08:00
|
||||
🔖 **ID**: 5b72b8e1-45ce-42fc-94be-854b4917138f"
|
||||
|
||||
if [ "$ISSUE_COUNT" -gt 0 ]; then
|
||||
log "📤 Telegram告警已发送"
|
||||
fi
|
||||
else
|
||||
log "✅ 系统健康，无需告警"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==================== 启动 ====================
|
||||
main "$@"
|
||||
Reference in new issue
Block a user