239 lines
8.6 KiB
Bash
Executable File
239 lines
8.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# Optimized CPU & Memory Monitoring System v2.0
|
|
# Heartbeat: 10 seconds (light), 60 seconds (heavy)
|
|
# Features: Service health check, threshold alerts, session file monitoring
|
|
# Optimizations: Reduced frequency, /proc instead of top, separation of concerns
|
|
|
|
LOG_DIR="/var/log/openclaw-monitor"
|
|
LOG_FILE="${LOG_FILE:-$LOG_DIR/system-metrics.log}"
|
|
ALERT_LOG="${LOG_DIR}/alerts.log"
|
|
SESSION_LOG="${LOG_DIR}/session-alerts.log"
|
|
MAX_RECORDS=5400
|
|
INTERVAL_LIGHT=10 # 轻量监控间隔（秒）
|
|
INTERVAL_HEAVY=60 # 重量监控间隔（秒）
|
|
OPENCLAW_PORTS=("18789")
|
|
|
|
# 资源阈值
|
|
CPU_THRESHOLD=80
|
|
MEM_THRESHOLD=80
|
|
LOAD_THRESHOLD=4.0
|
|
|
|
# 会话文件监控阈值
|
|
SESSION_DIR="/root/.openclaw/agents/main/sessions"
|
|
SESSION_WARN_SIZE=$((8 * 1024 * 1024)) # 8MB 警告
|
|
SESSION_NEW_SESSION_SIZE=$((5 * 1024 * 1024)) # 5MB 开新会话
|
|
SESSION_MARKER_FILE="/tmp/openclaw-session-warn.json"
|
|
SESSION_NEW_SESSION_FILE="/tmp/openclaw-session-new.json"
|
|
|
|
LAST_CPU=""
|
|
LAST_MEM=""
|
|
LAST_LOAD=""
|
|
LAST_SESSION_SIZE=0
|
|
LAST_SESSION_CHECK=0
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# 轻量监控：CPU/内存/负载（高频）
|
|
check_light() {
|
|
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
|
|
# CPU 使用率（使用 /proc/stat 更轻量）
|
|
local cpu_line=$(cat /proc/stat 2>/dev/null | grep '^cpu ' | head -1)
|
|
if [ -n "$cpu_line" ]; then
|
|
local cpu_idle=$(echo "$cpu_line" | awk '{print $5}')
|
|
# 简单估算：假设总时间是 idle + user + nice + system + iowait + irq + softirq
|
|
local cpu_total=$(echo "$cpu_line" | awk '{print $2+$3+$4+$5+$6+$7+$8}')
|
|
local cpu_usage="0.0"
|
|
if [ -n "$cpu_total" ] && [ "$cpu_total" != "0" ]; then
|
|
cpu_usage=$(awk "BEGIN {printf \"%.1f\", (($cpu_total - $cpu_idle) / $cpu_total) * 100}")
|
|
fi
|
|
else
|
|
# 回退到 top（极少情况）
|
|
cpu_usage=$(top -bn1 2>/dev/null | awk '/Cpu\(s\)/ {gsub(/,/, "", $2); print $2}' | head -1)
|
|
fi
|
|
|
|
# 内存信息
|
|
local mem_info=$(free -m 2>/dev/null | awk 'NR==2 {print $2,$3,$7}')
|
|
local mem_total=$(echo "$mem_info" | awk '{print $1}')
|
|
local mem_used=$(echo "$mem_info" | awk '{print $2}')
|
|
local mem_avail=$(echo "$mem_info" | awk '{print $3}')
|
|
local mem_usage="0.0"
|
|
if [ -n "$mem_total" ] && [ "$mem_total" != "0" ]; then
|
|
mem_usage=$(awk "BEGIN {printf \"%.1f\", ($mem_used)*100/$mem_total}")
|
|
fi
|
|
|
|
# 负载
|
|
local load_1m=$(cat /proc/loadavg | awk '{print $1}')
|
|
|
|
# 记录轻量日志
|
|
echo "LIGHT|${timestamp}|${cpu_usage}|${mem_usage}|${load_1m}" >> "$LOG_FILE"
|
|
|
|
# 检查阈值
|
|
check_thresholds "$timestamp" "$cpu_usage" "$mem_usage" "$load_1m"
|
|
}
|
|
|
|
# 重量监控：磁盘/会话文件（低频）
|
|
check_heavy() {
|
|
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
|
|
# 交换空间
|
|
local swap_info=$(free -m 2>/dev/null | awk 'NR==3 {print $2,$3}')
|
|
local swap_total=$(echo "$swap_info" | awk '{print $1}')
|
|
local swap_used=$(echo "$swap_info" | awk '{print $2}')
|
|
local swap_usage="0.0"
|
|
if [ -n "$swap_total" ] && [ "$swap_total" != "0" ]; then
|
|
swap_usage=$(awk "BEGIN {printf \"%.1f\", ($swap_used)*100/$swap_total}")
|
|
fi
|
|
|
|
# 负载（完整）
|
|
local load_5m=$(cat /proc/loadavg | awk '{print $2}')
|
|
local load_15m=$(cat /proc/loadavg | awk '{print $3}')
|
|
|
|
# 磁盘
|
|
local disk_usage=$(df -h / 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')
|
|
|
|
# OpenClaw 内存
|
|
local openclaw_mem="0.0"
|
|
local pids=$(pgrep -f "openclaw-gateway" 2>/dev/null)
|
|
if [ -n "$pids" ]; then
|
|
local sum=0
|
|
for pid in $pids; do
|
|
local rss=$(ps -o rss= -p "$pid" 2>/dev/null)
|
|
sum=$((sum + rss))
|
|
done
|
|
openclaw_mem=$(awk "BEGIN {printf \"%.1f\", $sum/1024}")
|
|
fi
|
|
|
|
# 服务状态
|
|
local service_status="DOWN"
|
|
for port in "${OPENCLAW_PORTS[@]}"; do
|
|
if ss -tlnp 2>/dev/null | grep -q ":${port} " || netstat -tlnp 2>/dev/null | grep -q ":${port} "; then
|
|
service_status="UP"
|
|
break
|
|
fi
|
|
done
|
|
|
|
# 记录重量日志
|
|
echo "HEAVY|${timestamp}|${swap_usage}|${load_5m}|${load_15m}|${disk_usage}|${openclaw_mem}|${service_status}" >> "$LOG_FILE"
|
|
|
|
# 检查服务状态
|
|
if [ "$service_status" = "DOWN" ]; then
|
|
echo "[CRITICAL] ${timestamp} OpenClaw SERVICE DOWN!" >> "$ALERT_LOG"
|
|
fi
|
|
|
|
# 检查会话文件（每60秒）
|
|
check_session_files "$timestamp"
|
|
}
|
|
|
|
check_session_files() {
|
|
if [ ! -d "$SESSION_DIR" ]; then
|
|
return
|
|
fi
|
|
|
|
# 获取当前主会话文件（最大文件）
|
|
local largest_file=""
|
|
local largest_size=0
|
|
local current_session_id=""
|
|
|
|
for file in "$SESSION_DIR"/*.jsonl; do
|
|
if [ -f "$file" ]; then
|
|
local size=$(stat -c%s "$file" 2>/dev/null || echo 0)
|
|
if [ "$size" -gt "$largest_size" ]; then
|
|
largest_size="$size"
|
|
largest_file="$file"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$largest_size" -eq 0 ] || [ -z "$largest_file" ]; then
|
|
return
|
|
fi
|
|
|
|
# 提取会话ID
|
|
current_session_id=$(basename "$largest_file" .jsonl)
|
|
|
|
# 8MB 警告
|
|
if [ "$largest_size" -ge "$SESSION_WARN_SIZE" ]; then
|
|
local size_mb=$(awk "BEGIN {printf \"%.1f\", $largest_size/1024/1024}")
|
|
local last_alert_size=$(cat "$SESSION_MARKER_FILE" 2>/dev/null | grep -o "\"warn\":[0-9]*" | grep -o "[0-9]*" || echo "0")
|
|
alert_threshold=$((last_alert_size / 1024 / 1024 + 1))
|
|
current_mb=$(echo "scale=1; $largest_size/1024/1024" | bc 2>/dev/null || echo "0")
|
|
if [ -n "$current_mb" ] && [ "$(echo "$current_mb >= $alert_threshold" | bc 2>/dev/null || echo "0")" -eq 1 ]; then
|
|
echo "[SESSION WARN] ${timestamp} 会话过大: ${current_session_id} (${size_mb}MB)" >> "$SESSION_LOG"
|
|
bash /root/.openclaw/scripts/send-session-alert.sh "warn" "$current_session_id" "$size_mb"
|
|
echo "{\"warn\":$largest_size,\"time\":\"$timestamp\"}" > "$SESSION_MARKER_FILE"
|
|
fi
|
|
fi
|
|
|
|
# 5MB 自动开新会话
|
|
if [ "$largest_size" -ge "$SESSION_NEW_SESSION_SIZE" ] && [ "$largest_size" -lt "$SESSION_WARN_SIZE" ]; then
|
|
local size_mb=$(awk "BEGIN {printf \"%.1f\", $largest_size/1024/1024}")
|
|
local last_processed=$(cat "$SESSION_NEW_SESSION_FILE" 2>/dev/null | grep -o "\"processed\":\"[^\"]*\"" | cut -d'"' -f4 || echo "")
|
|
if [ "$current_session_id" != "$last_processed" ]; then
|
|
echo "[SESSION NEW] ${timestamp} 会话超5MB: ${current_session_id} (${size_mb}MB)" >> "$SESSION_LOG"
|
|
bash /root/.openclaw/scripts/send-session-alert.sh "new" "$current_session_id" "$size_mb"
|
|
echo "{\"warn\":0,\"processed\":\"$current_session_id\",\"time\":\"$timestamp\"}" > "$SESSION_NEW_SESSION_FILE"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
check_thresholds() {
|
|
local ts="$1" cpu="$2" mem="$3" load="$4"
|
|
|
|
# CPU 检查
|
|
cpu=$(echo "$cpu" | tr -d ' ')
|
|
if [ -n "$cpu" ] && [ "$cpu" != "0.0" ]; then
|
|
local cpu_high=$(awk "BEGIN {print ($cpu > $CPU_THRESHOLD) ? 1 : 0}")
|
|
if [ "$cpu_high" = "1" ] && [ "$LAST_CPU" != "HIGH" ]; then
|
|
echo "[ALERT] ${ts} CPU HIGH: ${cpu}% (threshold: ${CPU_THRESHOLD}%)" >> "$ALERT_LOG"
|
|
fi
|
|
LAST_CPU="$([ "$cpu_high" = "1" ] && echo "HIGH" || echo "NORMAL")"
|
|
fi
|
|
|
|
# 内存检查
|
|
if [ -n "$mem" ] && [ "$mem" != "0.0" ]; then
|
|
local mem_high=$(awk "BEGIN {print ($mem > $MEM_THRESHOLD) ? 1 : 0}")
|
|
if [ "$mem_high" = "1" ] && [ "$LAST_MEM" != "HIGH" ]; then
|
|
echo "[ALERT] ${ts} MEM HIGH: ${mem}%" >> "$ALERT_LOG"
|
|
fi
|
|
LAST_MEM="$([ "$mem_high" = "1" ] && echo "HIGH" || echo "NORMAL")"
|
|
fi
|
|
|
|
# 负载检查
|
|
if [ -n "$load" ]; then
|
|
local load_high=$(awk "BEGIN {print ($load > $LOAD_THRESHOLD) ? 1 : 0}")
|
|
if [ "$load_high" = "1" ] && [ "$LAST_LOAD" != "HIGH" ]; then
|
|
echo "[ALERT] ${ts} LOAD HIGH: ${load} (threshold: ${LOAD_THRESHOLD})" >> "$ALERT_LOG"
|
|
fi
|
|
LAST_LOAD="$([ "$load_high" = "1" ] && echo "HIGH" || echo "NORMAL")"
|
|
fi
|
|
}
|
|
|
|
cleanup() {
|
|
local line_count=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
|
|
if [ "$line_count" -gt "$MAX_RECORDS" ]; then
|
|
local to_remove=$((line_count - MAX_RECORDS))
|
|
sed -i "1,${to_remove}d" "$LOG_FILE" 2>/dev/null
|
|
fi
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') 监控启动 (light:${INTERVAL_LIGHT}s, heavy:${INTERVAL_HEAVY}s)"
|
|
|
|
counter=0
|
|
while true; do
|
|
# 轻量监控每次都执行
|
|
check_light
|
|
|
|
# 计数器：每6次（60秒）执行一次重量监控
|
|
counter=$((counter + 1))
|
|
if [ $counter -ge 6 ]; then
|
|
check_heavy
|
|
counter=0
|
|
fi
|
|
|
|
cleanup
|
|
sleep $INTERVAL_LIGHT
|
|
done
|