每日备份 2026-06-10 12:57:13
This commit is contained in:
root-backup committed
1 parent
fda2ccb8c2
commit
3d2e638c6a
244 files changed
+58890
-3
No files matched your search
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/bin/bash
|
||||
# Optimized CPU & Memory Monitoring System v2.0
|
||||
# Heartbeat: 10 seconds (light), 60 seconds (heavy)
|
||||
# Features: Service health check, threshold alerts, session file monitoring
|
||||
# Optimizations: Reduced frequency, /proc instead of top, separation of concerns
|
||||
|
||||
LOG_DIR="/var/log/openclaw-monitor"
|
||||
LOG_FILE="${LOG_FILE:-$LOG_DIR/system-metrics.log}"
|
||||
ALERT_LOG="${LOG_DIR}/alerts.log"
|
||||
SESSION_LOG="${LOG_DIR}/session-alerts.log"
|
||||
MAX_RECORDS=5400
|
||||
INTERVAL_LIGHT=10 # 轻量监控间隔（秒）
|
||||
INTERVAL_HEAVY=60 # 重量监控间隔（秒）
|
||||
OPENCLAW_PORTS=("18789")
|
||||
|
||||
# 资源阈值
|
||||
CPU_THRESHOLD=80
|
||||
MEM_THRESHOLD=80
|
||||
LOAD_THRESHOLD=4.0
|
||||
|
||||
# 会话文件监控阈值
|
||||
SESSION_DIR="/root/.openclaw/agents/main/sessions"
|
||||
SESSION_WARN_SIZE=$((8 * 1024 * 1024)) # 8MB 警告
|
||||
SESSION_NEW_SESSION_SIZE=$((5 * 1024 * 1024)) # 5MB 开新会话
|
||||
SESSION_MARKER_FILE="/tmp/openclaw-session-warn.json"
|
||||
SESSION_NEW_SESSION_FILE="/tmp/openclaw-session-new.json"
|
||||
|
||||
LAST_CPU=""
|
||||
LAST_MEM=""
|
||||
LAST_LOAD=""
|
||||
LAST_SESSION_SIZE=0
|
||||
LAST_SESSION_CHECK=0
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# 轻量监控：CPU/内存/负载（高频）
|
||||
check_light() {
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# CPU 使用率（使用 /proc/stat 更轻量）
|
||||
local cpu_line=$(cat /proc/stat 2>/dev/null | grep '^cpu ' | head -1)
|
||||
if [ -n "$cpu_line" ]; then
|
||||
local cpu_idle=$(echo "$cpu_line" | awk '{print $5}')
|
||||
# 简单估算：假设总时间是 idle + user + nice + system + iowait + irq + softirq
|
||||
local cpu_total=$(echo "$cpu_line" | awk '{print $2+$3+$4+$5+$6+$7+$8}')
|
||||
local cpu_usage="0.0"
|
||||
if [ -n "$cpu_total" ] && [ "$cpu_total" != "0" ]; then
|
||||
cpu_usage=$(awk "BEGIN {printf \"%.1f\", (($cpu_total - $cpu_idle) / $cpu_total) * 100}")
|
||||
fi
|
||||
else
|
||||
# 回退到 top（极少情况）
|
||||
cpu_usage=$(top -bn1 2>/dev/null | awk '/Cpu\(s\)/ {gsub(/,/, "", $2); print $2}' | head -1)
|
||||
fi
|
||||
|
||||
# 内存信息
|
||||
local mem_info=$(free -m 2>/dev/null | awk 'NR==2 {print $2,$3,$7}')
|
||||
local mem_total=$(echo "$mem_info" | awk '{print $1}')
|
||||
local mem_used=$(echo "$mem_info" | awk '{print $2}')
|
||||
local mem_avail=$(echo "$mem_info" | awk '{print $3}')
|
||||
local mem_usage="0.0"
|
||||
if [ -n "$mem_total" ] && [ "$mem_total" != "0" ]; then
|
||||
mem_usage=$(awk "BEGIN {printf \"%.1f\", ($mem_used)*100/$mem_total}")
|
||||
fi
|
||||
|
||||
# 负载
|
||||
local load_1m=$(cat /proc/loadavg | awk '{print $1}')
|
||||
|
||||
# 记录轻量日志
|
||||
echo "LIGHT|${timestamp}|${cpu_usage}|${mem_usage}|${load_1m}" >> "$LOG_FILE"
|
||||
|
||||
# 检查阈值
|
||||
check_thresholds "$timestamp" "$cpu_usage" "$mem_usage" "$load_1m"
|
||||
}
|
||||
|
||||
# 重量监控：磁盘/会话文件（低频）
|
||||
check_heavy() {
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 交换空间
|
||||
local swap_info=$(free -m 2>/dev/null | awk 'NR==3 {print $2,$3}')
|
||||
local swap_total=$(echo "$swap_info" | awk '{print $1}')
|
||||
local swap_used=$(echo "$swap_info" | awk '{print $2}')
|
||||
local swap_usage="0.0"
|
||||
if [ -n "$swap_total" ] && [ "$swap_total" != "0" ]; then
|
||||
swap_usage=$(awk "BEGIN {printf \"%.1f\", ($swap_used)*100/$swap_total}")
|
||||
fi
|
||||
|
||||
# 负载（完整）
|
||||
local load_5m=$(cat /proc/loadavg | awk '{print $2}')
|
||||
local load_15m=$(cat /proc/loadavg | awk '{print $3}')
|
||||
|
||||
# 磁盘
|
||||
local disk_usage=$(df -h / 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
|
||||
# OpenClaw 内存
|
||||
local openclaw_mem="0.0"
|
||||
local pids=$(pgrep -f "openclaw-gateway" 2>/dev/null)
|
||||
if [ -n "$pids" ]; then
|
||||
local sum=0
|
||||
for pid in $pids; do
|
||||
local rss=$(ps -o rss= -p "$pid" 2>/dev/null)
|
||||
sum=$((sum + rss))
|
||||
done
|
||||
openclaw_mem=$(awk "BEGIN {printf \"%.1f\", $sum/1024}")
|
||||
fi
|
||||
|
||||
# 服务状态
|
||||
local service_status="DOWN"
|
||||
for port in "${OPENCLAW_PORTS[@]}"; do
|
||||
if ss -tlnp 2>/dev/null | grep -q ":${port} " || netstat -tlnp 2>/dev/null | grep -q ":${port} "; then
|
||||
service_status="UP"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# 记录重量日志
|
||||
echo "HEAVY|${timestamp}|${swap_usage}|${load_5m}|${load_15m}|${disk_usage}|${openclaw_mem}|${service_status}" >> "$LOG_FILE"
|
||||
|
||||
# 检查服务状态
|
||||
if [ "$service_status" = "DOWN" ]; then
|
||||
echo "[CRITICAL] ${timestamp} OpenClaw SERVICE DOWN!" >> "$ALERT_LOG"
|
||||
fi
|
||||
|
||||
# 检查会话文件（每60秒）
|
||||
check_session_files "$timestamp"
|
||||
}
|
||||
|
||||
check_session_files() {
|
||||
if [ ! -d "$SESSION_DIR" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# 获取当前主会话文件（最大文件）
|
||||
local largest_file=""
|
||||
local largest_size=0
|
||||
local current_session_id=""
|
||||
|
||||
for file in "$SESSION_DIR"/*.jsonl; do
|
||||
if [ -f "$file" ]; then
|
||||
local size=$(stat -c%s "$file" 2>/dev/null || echo 0)
|
||||
if [ "$size" -gt "$largest_size" ]; then
|
||||
largest_size="$size"
|
||||
largest_file="$file"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$largest_size" -eq 0 ] || [ -z "$largest_file" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# 提取会话ID
|
||||
current_session_id=$(basename "$largest_file" .jsonl)
|
||||
|
||||
# 8MB 警告
|
||||
if [ "$largest_size" -ge "$SESSION_WARN_SIZE" ]; then
|
||||
local size_mb=$(awk "BEGIN {printf \"%.1f\", $largest_size/1024/1024}")
|
||||
local last_alert_size=$(cat "$SESSION_MARKER_FILE" 2>/dev/null | grep -o "\"warn\":[0-9]*" | grep -o "[0-9]*" || echo "0")
|
||||
alert_threshold=$((last_alert_size / 1024 / 1024 + 1))
|
||||
current_mb=$(echo "scale=1; $largest_size/1024/1024" | bc 2>/dev/null || echo "0")
|
||||
if [ -n "$current_mb" ] && [ "$(echo "$current_mb >= $alert_threshold" | bc 2>/dev/null || echo "0")" -eq 1 ]; then
|
||||
echo "[SESSION WARN] ${timestamp} 会话过大: ${current_session_id} (${size_mb}MB)" >> "$SESSION_LOG"
|
||||
bash /root/.openclaw/scripts/send-session-alert.sh "warn" "$current_session_id" "$size_mb"
|
||||
echo "{\"warn\":$largest_size,\"time\":\"$timestamp\"}" > "$SESSION_MARKER_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 5MB 自动开新会话
|
||||
if [ "$largest_size" -ge "$SESSION_NEW_SESSION_SIZE" ] && [ "$largest_size" -lt "$SESSION_WARN_SIZE" ]; then
|
||||
local size_mb=$(awk "BEGIN {printf \"%.1f\", $largest_size/1024/1024}")
|
||||
local last_processed=$(cat "$SESSION_NEW_SESSION_FILE" 2>/dev/null | grep -o "\"processed\":\"[^\"]*\"" | cut -d'"' -f4 || echo "")
|
||||
if [ "$current_session_id" != "$last_processed" ]; then
|
||||
echo "[SESSION NEW] ${timestamp} 会话超5MB: ${current_session_id} (${size_mb}MB)" >> "$SESSION_LOG"
|
||||
bash /root/.openclaw/scripts/send-session-alert.sh "new" "$current_session_id" "$size_mb"
|
||||
echo "{\"warn\":0,\"processed\":\"$current_session_id\",\"time\":\"$timestamp\"}" > "$SESSION_NEW_SESSION_FILE"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
check_thresholds() {
|
||||
local ts="$1" cpu="$2" mem="$3" load="$4"
|
||||
|
||||
# CPU 检查
|
||||
cpu=$(echo "$cpu" | tr -d ' ')
|
||||
if [ -n "$cpu" ] && [ "$cpu" != "0.0" ]; then
|
||||
local cpu_high=$(awk "BEGIN {print ($cpu > $CPU_THRESHOLD) ? 1 : 0}")
|
||||
if [ "$cpu_high" = "1" ] && [ "$LAST_CPU" != "HIGH" ]; then
|
||||
echo "[ALERT] ${ts} CPU HIGH: ${cpu}% (threshold: ${CPU_THRESHOLD}%)" >> "$ALERT_LOG"
|
||||
fi
|
||||
LAST_CPU="$([ "$cpu_high" = "1" ] && echo "HIGH" || echo "NORMAL")"
|
||||
fi
|
||||
|
||||
# 内存检查
|
||||
if [ -n "$mem" ] && [ "$mem" != "0.0" ]; then
|
||||
local mem_high=$(awk "BEGIN {print ($mem > $MEM_THRESHOLD) ? 1 : 0}")
|
||||
if [ "$mem_high" = "1" ] && [ "$LAST_MEM" != "HIGH" ]; then
|
||||
echo "[ALERT] ${ts} MEM HIGH: ${mem}%" >> "$ALERT_LOG"
|
||||
fi
|
||||
LAST_MEM="$([ "$mem_high" = "1" ] && echo "HIGH" || echo "NORMAL")"
|
||||
fi
|
||||
|
||||
# 负载检查
|
||||
if [ -n "$load" ]; then
|
||||
local load_high=$(awk "BEGIN {print ($load > $LOAD_THRESHOLD) ? 1 : 0}")
|
||||
if [ "$load_high" = "1" ] && [ "$LAST_LOAD" != "HIGH" ]; then
|
||||
echo "[ALERT] ${ts} LOAD HIGH: ${load} (threshold: ${LOAD_THRESHOLD})" >> "$ALERT_LOG"
|
||||
fi
|
||||
LAST_LOAD="$([ "$load_high" = "1" ] && echo "HIGH" || echo "NORMAL")"
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local line_count=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
if [ "$line_count" -gt "$MAX_RECORDS" ]; then
|
||||
local to_remove=$((line_count - MAX_RECORDS))
|
||||
sed -i "1,${to_remove}d" "$LOG_FILE" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') 监控启动 (light:${INTERVAL_LIGHT}s, heavy:${INTERVAL_HEAVY}s)"
|
||||
|
||||
counter=0
|
||||
while true; do
|
||||
# 轻量监控每次都执行
|
||||
check_light
|
||||
|
||||
# 计数器：每6次（60秒）执行一次重量监控
|
||||
counter=$((counter + 1))
|
||||
if [ $counter -ge 6 ]; then
|
||||
check_heavy
|
||||
counter=0
|
||||
fi
|
||||
|
||||
cleanup
|
||||
sleep $INTERVAL_LIGHT
|
||||
done
|
||||
Reference in new issue
Block a user