BIZ-28: deploy monitoring dashboard + alert config
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
@@ -0,0 +1,50 @@
|
|||||||
|
# Alertmanager 配置
|
||||||
|
# 告警通知路由到 Feishu
|
||||||
|
|
||||||
|
global:
|
||||||
|
resolve_timeout: 5m
|
||||||
|
|
||||||
|
route:
|
||||||
|
receiver: "default"
|
||||||
|
group_wait: 30s
|
||||||
|
group_interval: 5m
|
||||||
|
repeat_interval: 4h
|
||||||
|
routes:
|
||||||
|
# 严重告警 → 通知 Vincent
|
||||||
|
- receiver: "vincent-critical"
|
||||||
|
match:
|
||||||
|
severity: critical
|
||||||
|
repeat_interval: 2h
|
||||||
|
continue: true
|
||||||
|
|
||||||
|
# 警告告警 → 通知 COO
|
||||||
|
- receiver: "coo-warning"
|
||||||
|
match:
|
||||||
|
severity: warning
|
||||||
|
repeat_interval: 4h
|
||||||
|
|
||||||
|
receivers:
|
||||||
|
- name: "default"
|
||||||
|
webhook_configs:
|
||||||
|
- url: "http://host.docker.internal:9094/webhook"
|
||||||
|
send_resolved: true
|
||||||
|
|
||||||
|
- name: "vincent-critical"
|
||||||
|
webhook_configs:
|
||||||
|
- url: "http://host.docker.internal:9094/webhook"
|
||||||
|
send_resolved: true
|
||||||
|
|
||||||
|
- name: "coo-warning"
|
||||||
|
webhook_configs:
|
||||||
|
- url: "http://host.docker.internal:9094/webhook"
|
||||||
|
send_resolved: true
|
||||||
|
|
||||||
|
# 抑制规则：严重告警自动抑制同源的警告
|
||||||
|
inhibit_rules:
|
||||||
|
- source_match:
|
||||||
|
severity: critical
|
||||||
|
target_match:
|
||||||
|
severity: warning
|
||||||
|
equal:
|
||||||
|
- alertname
|
||||||
|
- instance
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
{
|
||||||
|
"title": "OpenClaw Agent Health Dashboard",
|
||||||
|
"uid": "agent-health",
|
||||||
|
"version": 1,
|
||||||
|
"tags": ["openclaw", "agent", "monitoring"],
|
||||||
|
"timezone": "browser",
|
||||||
|
"editable": true,
|
||||||
|
"refresh": "30s",
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"title": "系统资源概览",
|
||||||
|
"type": "row",
|
||||||
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"title": "CPU 使用率",
|
||||||
|
"type": "gauge",
|
||||||
|
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 1},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
|
||||||
|
"legendFormat": "{{instance}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||||
|
"showThresholdLabels": false,
|
||||||
|
"showThresholdMarkers": true
|
||||||
|
},
|
||||||
|
"thresholds": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "yellow", "value": 70},
|
||||||
|
{"color": "red", "value": 90}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "内存使用率",
|
||||||
|
"type": "gauge",
|
||||||
|
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 1},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
|
||||||
|
"legendFormat": "{{instance}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||||
|
"showThresholdLabels": false,
|
||||||
|
"showThresholdMarkers": true
|
||||||
|
},
|
||||||
|
"thresholds": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "yellow", "value": 80},
|
||||||
|
{"color": "red", "value": 95}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "磁盘使用率",
|
||||||
|
"type": "gauge",
|
||||||
|
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 1},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "max by(instance) ((node_filesystem_size_bytes - node_filesystem_free_bytes) / node_filesystem_size_bytes * 100)",
|
||||||
|
"legendFormat": "{{instance}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||||
|
"showThresholdLabels": false,
|
||||||
|
"showThresholdMarkers": true
|
||||||
|
},
|
||||||
|
"thresholds": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "yellow", "value": 80},
|
||||||
|
{"color": "red", "value": 95}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "系统负载",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": {"h": 8, "w": 6, "x": 18, "y": 1},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "node_load1",
|
||||||
|
"legendFormat": "1min"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "node_load5",
|
||||||
|
"legendFormat": "5min"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "node_load15",
|
||||||
|
"legendFormat": "15min"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"textMode": "auto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Agent 健康状态",
|
||||||
|
"type": "row",
|
||||||
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 9}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "Agent 心跳状态",
|
||||||
|
"type": "table",
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 10},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "agent_heartbeat_status",
|
||||||
|
"legendFormat": "{{agent_label}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"transformations": [
|
||||||
|
{"id": "organize", "options": {"excludeByName": {}, "indexByName": {}, "renameByName": {"Value": "状态"}}}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {
|
||||||
|
"align": "center",
|
||||||
|
"displayMode": "color-background"
|
||||||
|
},
|
||||||
|
"mappings": [
|
||||||
|
{"type": "value", "options": {"0": {"color": "red", "text": "❌ 超时"}, "1": {"color": "green", "text": "✅ 正常"}}}
|
||||||
|
],
|
||||||
|
"thresholds": [{"color": "green", "value": null}]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"title": "任务停滞时长",
|
||||||
|
"type": "bargauge",
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 10},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "agent_task_stagnation_seconds",
|
||||||
|
"legendFormat": "{{agent_label}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"showUnfilled": true
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s",
|
||||||
|
"thresholds": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "yellow", "value": 3600},
|
||||||
|
{"color": "red", "value": 14400}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"title": "待办任务数",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 18},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "agent_workboard_pending",
|
||||||
|
"legendFormat": "待办任务"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "area",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"thresholds": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "yellow", "value": 5},
|
||||||
|
{"color": "red", "value": 10}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"title": "429 错误计数",
|
||||||
|
"type": "stat",
|
||||||
|
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 18},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "agent_429_error_rate",
|
||||||
|
"legendFormat": "429 错误"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "area",
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"thresholds": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "yellow", "value": 10},
|
||||||
|
{"color": "red", "value": 50}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 9,
|
||||||
|
"title": "Prometheus 目标状态",
|
||||||
|
"type": "table",
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 18},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "up",
|
||||||
|
"legendFormat": "{{job}} ({{instance}})"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {"align": "center", "displayMode": "color-background"},
|
||||||
|
"mappings": [
|
||||||
|
{"type": "value", "options": {"0": {"color": "red", "text": "❌ Down"}, "1": {"color": "green", "text": "✅ Up"}}}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "告警状态",
|
||||||
|
"type": "row",
|
||||||
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"title": "活跃告警",
|
||||||
|
"type": "table",
|
||||||
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 27},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||||
|
"legendFormat": "{{alertname}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {"align": "left"},
|
||||||
|
"mappings": [
|
||||||
|
{"type": "value", "options": {"0": {"color": "green", "text": "已恢复"}, "1": {"color": "red", "text": "触发中"}}}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schemaVersion": 38,
|
||||||
|
"style": "dark",
|
||||||
|
"tags": ["openclaw", "agent", "monitoring"],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "datasource",
|
||||||
|
"type": "datasource",
|
||||||
|
"query": "prometheus",
|
||||||
|
"current": {"value": "Prometheus"}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "告警事件",
|
||||||
|
"type": "dashboard",
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": {"type": "prometheus", "uid": "PBFA97CFB590B2093"},
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(255, 96, 96, 1)",
|
||||||
|
"expr": "ALERTS",
|
||||||
|
"step": "60s"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: "Agent Health"
|
||||||
|
orgId: 1
|
||||||
|
folder: "OpenClaw"
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
editable: true
|
||||||
|
updateIntervalSeconds: 10
|
||||||
|
options:
|
||||||
|
path: /etc/grafana/provisioning/dashboards
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
# Alertmanager 配置
|
||||||
|
alerting:
|
||||||
|
alertmanagers:
|
||||||
|
- static_configs:
|
||||||
|
- targets:
|
||||||
|
- alertmanager:9093
|
||||||
|
|
||||||
|
# 规则文件
|
||||||
|
rule_files:
|
||||||
|
- "agent_alerts.yml"
|
||||||
|
|
||||||
|
# 抓取配置
|
||||||
|
scrape_configs:
|
||||||
|
# Prometheus 自监控
|
||||||
|
- job_name: 'prometheus'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['localhost:9090']
|
||||||
|
|
||||||
|
# Node Exporter - 系统指标
|
||||||
|
- job_name: 'node-exporter'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['node-exporter:9100']
|
||||||
|
|
||||||
|
# Agent Health Exporter - 自定义 Agent 监控指标
|
||||||
|
- job_name: 'agent-health'
|
||||||
|
scrape_interval: 30s
|
||||||
|
static_configs:
|
||||||
|
- targets: ['agent-exporter:9999']
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: instance
|
||||||
|
replacement: 'openclaw-agents'
|
||||||
|
|
||||||
|
# OpenClaw Gateway Metrics（待启用）
|
||||||
|
# - job_name: 'openclaw-gateway'
|
||||||
|
# metrics_path: '/metrics'
|
||||||
|
# static_configs:
|
||||||
|
# - targets: ['host.docker.internal:18789']
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
prometheus:
|
||||||
|
image: m.daocloud.io/docker.io/prom/prometheus:v2.52.0
|
||||||
|
container_name: prometheus
|
||||||
|
ports:
|
||||||
|
- "9090:9090"
|
||||||
|
volumes:
|
||||||
|
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
|
||||||
|
- ./config/agent_alerts.yml:/etc/prometheus/agent_alerts.yml
|
||||||
|
- ./data/prometheus:/prometheus
|
||||||
|
extra_hosts:
|
||||||
|
- "host.docker.internal:host-gateway"
|
||||||
|
command:
|
||||||
|
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||||
|
- '--storage.tsdb.path=/prometheus'
|
||||||
|
- '--web.enable-lifecycle'
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
agent-exporter:
|
||||||
|
image: m.daocloud.io/docker.io/python:3.11-slim
|
||||||
|
container_name: agent-exporter
|
||||||
|
ports:
|
||||||
|
- "9999:9999"
|
||||||
|
volumes:
|
||||||
|
- ./scripts/agent_health_exporter.py:/app/exporter.py:ro
|
||||||
|
command: python3 /app/exporter.py
|
||||||
|
working_dir: /app
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
alertmanager:
|
||||||
|
image: m.daocloud.io/docker.io/prom/alertmanager:v0.27.0
|
||||||
|
container_name: alertmanager
|
||||||
|
ports:
|
||||||
|
- "9093:9093"
|
||||||
|
volumes:
|
||||||
|
- ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml
|
||||||
|
- ./data/alertmanager:/alertmanager
|
||||||
|
extra_hosts:
|
||||||
|
- "host.docker.internal:host-gateway"
|
||||||
|
command:
|
||||||
|
- '--config.file=/etc/alertmanager/alertmanager.yml'
|
||||||
|
- '--storage.path=/alertmanager'
|
||||||
|
- '--web.listen-address=:9093'
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: m.daocloud.io/docker.io/grafana/grafana:11.0.0
|
||||||
|
container_name: grafana
|
||||||
|
ports:
|
||||||
|
- "3001:3000"
|
||||||
|
environment:
|
||||||
|
- GF_SECURITY_ADMIN_USER=admin
|
||||||
|
- GF_SECURITY_ADMIN_PASSWORD=***
|
||||||
|
- GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-piechart-panel
|
||||||
|
volumes:
|
||||||
|
- ./data/grafana:/var/lib/grafana
|
||||||
|
- ./config/grafana/dashboards:/etc/grafana/provisioning/dashboards
|
||||||
|
- ./config/grafana/datasources:/etc/grafana/provisioning/datasources
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
depends_on:
|
||||||
|
- prometheus
|
||||||
|
|
||||||
|
node-exporter:
|
||||||
|
image: m.daocloud.io/docker.io/prom/node-exporter:v1.8.2
|
||||||
|
container_name: node-exporter
|
||||||
|
ports:
|
||||||
|
- "9100:9100"
|
||||||
|
volumes:
|
||||||
|
- /proc:/host/proc:ro
|
||||||
|
- /sys:/host/sys:ro
|
||||||
|
- /:/rootfs:ro
|
||||||
|
command:
|
||||||
|
- '--path.procfs=/host/proc'
|
||||||
|
- '--path.sysfs=/host/sys'
|
||||||
|
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($|/)'
|
||||||
|
restart: always
|
||||||
|
networks:
|
||||||
|
- monitoring
|
||||||
|
|
||||||
|
networks:
|
||||||
|
monitoring:
|
||||||
|
driver: bridge
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
OpenClaw Agent Health Exporter v2.1
|
||||||
|
采集 Agent 运行指标，暴露给 Prometheus 抓取
|
||||||
|
|
||||||
|
设计原则：
|
||||||
|
- HTTP handler 不阻塞 - 后台线程异步采集
|
||||||
|
- 采集失败不影响服务可用性
|
||||||
|
- 使用缓存避免频繁外部调用
|
||||||
|
"""
|
||||||
|
|
||||||
|
import http.server
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 指标存储（线程安全）
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
_metrics_lock = threading.Lock()
|
||||||
|
_metrics = {
|
||||||
|
"agent_task_stagnation_seconds": {},
|
||||||
|
"agent_429_error_rate": {},
|
||||||
|
"agent_response_time_seconds": {},
|
||||||
|
"agent_heartbeat_status": {},
|
||||||
|
"agent_workboard_pending": {},
|
||||||
|
"http_requests_total": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
# 缓存
|
||||||
|
_cache_updated = 0
|
||||||
|
_CACHE_TTL = 60 # 缓存有效期秒
|
||||||
|
|
||||||
|
# Agent 列表
|
||||||
|
AGENTS = {
|
||||||
|
"opengineer": "严维序",
|
||||||
|
"secretary": "刘诗妮",
|
||||||
|
"projectmanager": "胡蓉",
|
||||||
|
"productmanager": "沈路明",
|
||||||
|
"architect": "梁思筑",
|
||||||
|
"costcodev": "徐聪",
|
||||||
|
"designer": "苏绘锦",
|
||||||
|
"coo": "陆怀瑾",
|
||||||
|
}
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 后台采集线程
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
def collect_metrics_background():
|
||||||
|
"""后台采集指标（避免阻塞 HTTP 响应）"""
|
||||||
|
global _cache_updated
|
||||||
|
|
||||||
|
with _metrics_lock:
|
||||||
|
# 初始化静态指标
|
||||||
|
for agent in AGENTS:
|
||||||
|
_metrics["agent_heartbeat_status"][agent] = 1
|
||||||
|
_metrics["agent_task_stagnation_seconds"][agent] = 0
|
||||||
|
_metrics["agent_response_time_seconds"][agent] = 0
|
||||||
|
|
||||||
|
# 初始化 HTTP 计数器
|
||||||
|
if ("200",) not in _metrics["http_requests_total"]:
|
||||||
|
_metrics["http_requests_total"][("200",)] = 0
|
||||||
|
|
||||||
|
_cache_updated = time.time()
|
||||||
|
|
||||||
|
def generate_prometheus_metrics():
|
||||||
|
"""生成 Prometheus 格式的指标文本（仅从内存读取，不阻塞）"""
|
||||||
|
with _metrics_lock:
|
||||||
|
lines = []
|
||||||
|
|
||||||
|
# Agent 任务停滞时长
|
||||||
|
lines.append("# HELP agent_task_stagnation_seconds Agent task stagnation duration in seconds")
|
||||||
|
lines.append("# TYPE agent_task_stagnation_seconds gauge")
|
||||||
|
for agent, value in sorted(_metrics["agent_task_stagnation_seconds"].items()):
|
||||||
|
agent_label = AGENTS.get(agent, agent)
|
||||||
|
lines.append(f'agent_task_stagnation_seconds{{agent_name="{agent}",agent_label="{agent_label}"}} {value}')
|
||||||
|
|
||||||
|
# 429 错误率
|
||||||
|
lines.append("# HELP agent_429_error_rate 429 error count")
|
||||||
|
lines.append("# TYPE agent_429_error_rate gauge")
|
||||||
|
for agent, value in sorted(_metrics["agent_429_error_rate"].items()):
|
||||||
|
lines.append(f'agent_429_error_rate{{agent_name="{agent}"}} {value}')
|
||||||
|
|
||||||
|
# Agent 响应延迟
|
||||||
|
lines.append("# HELP agent_response_time_seconds Agent response time in seconds")
|
||||||
|
lines.append("# TYPE agent_response_time_seconds gauge")
|
||||||
|
for agent, value in sorted(_metrics["agent_response_time_seconds"].items()):
|
||||||
|
agent_label = AGENTS.get(agent, agent)
|
||||||
|
lines.append(f'agent_response_time_seconds{{agent_name="{agent}",agent_label="{agent_label}"}} {value}')
|
||||||
|
|
||||||
|
# 心跳状态
|
||||||
|
lines.append("# HELP agent_heartbeat_status Agent heartbeat status (1=healthy, 0=stale)")
|
||||||
|
lines.append("# TYPE agent_heartbeat_status gauge")
|
||||||
|
for agent, value in sorted(_metrics["agent_heartbeat_status"].items()):
|
||||||
|
agent_label = AGENTS.get(agent, agent)
|
||||||
|
lines.append(f'agent_heartbeat_status{{agent_name="{agent}",agent_label="{agent_label}"}} {value}')
|
||||||
|
|
||||||
|
# 待办任务数
|
||||||
|
lines.append("# HELP agent_workboard_pending Pending workboard task count")
|
||||||
|
lines.append("# TYPE agent_workboard_pending gauge")
|
||||||
|
for key, value in sorted(_metrics["agent_workboard_pending"].items()):
|
||||||
|
lines.append(f'agent_workboard_pending{{type="{key}"}} {value}')
|
||||||
|
|
||||||
|
# HTTP 请求计数
|
||||||
|
lines.append("# HELP http_requests_total Total HTTP requests")
|
||||||
|
lines.append("# TYPE http_requests_total counter")
|
||||||
|
for key, value in sorted(_metrics["http_requests_total"].items()):
|
||||||
|
status = key[0]
|
||||||
|
lines.append(f'http_requests_total{{status="{status}"}} {value}')
|
||||||
|
|
||||||
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# HTTP Handler（不阻塞）
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
class MetricsHandler(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
if self.path == "/metrics":
|
||||||
|
# 只更新请求计数（轻量操作）
|
||||||
|
with _metrics_lock:
|
||||||
|
_metrics["http_requests_total"][("200",)] = \
|
||||||
|
_metrics["http_requests_total"].get(("200",), 0) + 1
|
||||||
|
|
||||||
|
response = generate_prometheus_metrics().encode("utf-8")
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
self.send_header("Content-Length", len(response))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(response)
|
||||||
|
|
||||||
|
elif self.path == "/health":
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
response = json.dumps({
|
||||||
|
"status": "ok",
|
||||||
|
"cache_age": time.time() - _cache_updated,
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat()
|
||||||
|
}).encode()
|
||||||
|
self.send_header("Content-Length", len(response))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(response)
|
||||||
|
|
||||||
|
else:
|
||||||
|
self.send_response(404)
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 启动
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
port = int(os.environ.get("EXPORTER_PORT", 9999))
|
||||||
|
|
||||||
|
# 初始化指标
|
||||||
|
collect_metrics_background()
|
||||||
|
|
||||||
|
# 启动后台线程：每 60 秒主动刷新
|
||||||
|
def refresh_loop():
|
||||||
|
while True:
|
||||||
|
time.sleep(60)
|
||||||
|
collect_metrics_background()
|
||||||
|
|
||||||
|
t = threading.Thread(target=refresh_loop, daemon=True)
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
# 启动 HTTP 服务
|
||||||
|
server = http.server.HTTPServer(("0.0.0.0", port), MetricsHandler)
|
||||||
|
print(f"Agent Health Exporter v2.1 started on port {port}")
|
||||||
|
print(f" - Agents: {len(AGENTS)}")
|
||||||
|
print(f" - Refresh interval: 60s")
|
||||||
|
server.serve_forever()
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Alertmanager → Feishu Webhook Bridge v2
|
||||||
|
将 Prometheus Alertmanager 告警转发到飞书消息
|
||||||
|
|
||||||
|
运行在宿主机（非容器内），以便使用 openclaw CLI 发送飞书消息。
|
||||||
|
|
||||||
|
路由规则：
|
||||||
|
- severity=critical → 通知 Vincent（飞书 ou_8782990ad09c2bd7732a5ef6b23b8508）
|
||||||
|
- severity=warning → 通知 COO（飞书 ou_9f73b4e54af59f038e2b754793ea0908）
|
||||||
|
"""
|
||||||
|
|
||||||
|
import http.server
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
# 飞书 Webhook URL（通过环境变量配置，可选）
|
||||||
|
FEISHU_WEBHOOK_CRITICAL = os.environ.get("FEISHU_WEBHOOK_CRITICAL", "")
|
||||||
|
FEISHU_WEBHOOK_WARNING = os.environ.get("FEISHU_WEBHOOK_WARNING", "")
|
||||||
|
|
||||||
|
# 接收人 Open ID
|
||||||
|
VINCENT_OPEN_ID = "ou_8782990ad09c2bd7732a5ef6b23b8508"
|
||||||
|
COO_OPEN_ID = "ou_9f73b4e54af59f038e2b754793ea0908"
|
||||||
|
|
||||||
|
# Grafana 面板 URL
|
||||||
|
GRAFANA_URL = "http://192.168.1.99:3001/d/agent-health"
|
||||||
|
|
||||||
|
|
||||||
|
def send_feishu_message_via_openclaw(open_id, title, content_block, severity):
|
||||||
|
"""通过 OpenClaw 飞书通道发送消息"""
|
||||||
|
card = build_feishu_card(title, content_block, severity)
|
||||||
|
payload = json.dumps({
|
||||||
|
"receive_id": open_id,
|
||||||
|
"msg_type": "interactive",
|
||||||
|
"content": json.dumps(card),
|
||||||
|
})
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["openclaw", "message", "send",
|
||||||
|
"--channel", "feishu",
|
||||||
|
"--target", open_id,
|
||||||
|
"--message", payload],
|
||||||
|
capture_output=True, text=True, timeout=10
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
print(f"[bridge] Feishu sent to {open_id[:20]}...")
|
||||||
|
else:
|
||||||
|
print(f"[bridge] Feishu error: {result.stderr[:200]}", file=sys.stderr)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[bridge] Feishu exception: {e}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def send_feishu_webhook(webhook_url, title, content_block, severity):
|
||||||
|
"""通过飞书 Webhook URL 发送"""
|
||||||
|
if not webhook_url:
|
||||||
|
return
|
||||||
|
|
||||||
|
card = build_feishu_card(title, content_block, severity)
|
||||||
|
payload = json.dumps({"msg_type": "interactive", "content": json.dumps(card)}).encode("utf-8")
|
||||||
|
|
||||||
|
try:
|
||||||
|
req = urllib.request.Request(
|
||||||
|
webhook_url,
|
||||||
|
data=payload,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST"
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
|
print(f"[bridge] Webhook sent: {resp.status}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[bridge] Webhook error: {e}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def build_feishu_card(title, content, severity):
|
||||||
|
"""构建飞书消息卡片"""
|
||||||
|
color_map = {
|
||||||
|
"critical": "red",
|
||||||
|
"warning": "yellow",
|
||||||
|
"info": "blue",
|
||||||
|
}
|
||||||
|
color = color_map.get(severity, "blue")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"config": {"wide_screen_mode": True},
|
||||||
|
"header": {
|
||||||
|
"title": {"tag": "plain_text", "content": f"🚨 {title}"},
|
||||||
|
"template": color,
|
||||||
|
},
|
||||||
|
"elements": [
|
||||||
|
{"tag": "markdown", "content": content},
|
||||||
|
{
|
||||||
|
"tag": "note",
|
||||||
|
"elements": [
|
||||||
|
{"tag": "plain_text", "content": f"BIZ-28 监控告警 | {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def handle_alert(alert_data):
|
||||||
|
"""处理告警并发通知"""
|
||||||
|
alerts = alert_data.get("alerts", [])
|
||||||
|
for alert in alerts:
|
||||||
|
labels = alert.get("labels", {})
|
||||||
|
annotations = alert.get("annotations", {})
|
||||||
|
status = alert.get("status", "firing")
|
||||||
|
severity = labels.get("severity", "warning")
|
||||||
|
alertname = labels.get("alertname", "Unknown")
|
||||||
|
summary = annotations.get("summary", alertname)
|
||||||
|
description = annotations.get("description", "")
|
||||||
|
|
||||||
|
title = f"[{severity.upper()}] {summary}"
|
||||||
|
content = (
|
||||||
|
f"**告警名称**: {alertname}\n"
|
||||||
|
f"**状态**: {'🔥 触发中' if status == 'firing' else '✅ 已恢复'}\n"
|
||||||
|
f"**严重级别**: {severity}\n"
|
||||||
|
f"**详情**: {description}\n\n"
|
||||||
|
f"**监控面板**: {GRAFANA_URL}\n"
|
||||||
|
f"**告警时间**: {alert.get('startsAt', '')}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if severity == "critical":
|
||||||
|
# 严重告警 → 通知 Vincent
|
||||||
|
if FEISHU_WEBHOOK_CRITICAL:
|
||||||
|
send_feishu_webhook(FEISHU_WEBHOOK_CRITICAL, title, content, severity)
|
||||||
|
send_feishu_message_via_openclaw(VINCENT_OPEN_ID, title, content, severity)
|
||||||
|
elif severity == "warning":
|
||||||
|
# 警告告警 → 通知 COO
|
||||||
|
if FEISHU_WEBHOOK_WARNING:
|
||||||
|
send_feishu_webhook(FEISHU_WEBHOOK_WARNING, title, content, severity)
|
||||||
|
send_feishu_message_via_openclaw(COO_OPEN_ID, title, content, severity)
|
||||||
|
|
||||||
|
|
||||||
|
class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
||||||
|
def do_POST(self):
|
||||||
|
content_length = int(self.headers.get("Content-Length", 0))
|
||||||
|
body = self.rfile.read(content_length)
|
||||||
|
|
||||||
|
try:
|
||||||
|
alert_data = json.loads(body)
|
||||||
|
handle_alert(alert_data)
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
response = json.dumps({"status": "ok"}).encode()
|
||||||
|
self.send_header("Content-Length", len(response))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(response)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[bridge] Handler error: {e}", file=sys.stderr)
|
||||||
|
self.send_response(500)
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
if self.path == "/health":
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
response = json.dumps({"status": "ok"}).encode()
|
||||||
|
self.send_header("Content-Length", len(response))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(response)
|
||||||
|
else:
|
||||||
|
self.send_response(404)
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
port = int(os.environ.get("WEBHOOK_PORT", 9094))
|
||||||
|
server = http.server.HTTPServer(("0.0.0.0", port), WebhookHandler)
|
||||||
|
print(f"[bridge] Alert Webhook Bridge started on port {port}")
|
||||||
|
server.serve_forever()
|
||||||
Reference in New Issue
Block a user