Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 vincent
|
77f4eb1579 | ||
|
vincent
|
bea11d04fb | ||
|
vincent
|
3f08ae4004 | ||
|
vincent
|
b0cf98e422 | ||
|
vincent
|
a8fa922095 | ||
|
vincent
|
be24de9ced | ||
|
vincent
|
fed64cc279 |
@@ -1,50 +0,0 @@
|
||||
# Alertmanager 配置
|
||||
# 告警通知路由到 Feishu
|
||||
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: "default"
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
# 严重告警 → 通知 Vincent
|
||||
- receiver: "vincent-critical"
|
||||
match:
|
||||
severity: critical
|
||||
repeat_interval: 2h
|
||||
continue: true
|
||||
|
||||
# 警告告警 → 通知 COO
|
||||
- receiver: "coo-warning"
|
||||
match:
|
||||
severity: warning
|
||||
repeat_interval: 4h
|
||||
|
||||
receivers:
|
||||
- name: "default"
|
||||
webhook_configs:
|
||||
- url: "http://host.docker.internal:9094/webhook"
|
||||
send_resolved: true
|
||||
|
||||
- name: "vincent-critical"
|
||||
webhook_configs:
|
||||
- url: "http://host.docker.internal:9094/webhook"
|
||||
send_resolved: true
|
||||
|
||||
- name: "coo-warning"
|
||||
webhook_configs:
|
||||
- url: "http://host.docker.internal:9094/webhook"
|
||||
send_resolved: true
|
||||
|
||||
# 抑制规则：严重告警自动抑制同源的警告
|
||||
inhibit_rules:
|
||||
- source_match:
|
||||
severity: critical
|
||||
target_match:
|
||||
severity: warning
|
||||
equal:
|
||||
- alertname
|
||||
- instance
|
||||
@@ -1,288 +0,0 @@
|
||||
{
|
||||
"title": "OpenClaw Agent Health Dashboard",
|
||||
"uid": "agent-health",
|
||||
"version": 1,
|
||||
"tags": ["openclaw", "agent", "monitoring"],
|
||||
"timezone": "browser",
|
||||
"editable": true,
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"title": "系统资源概览",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"title": "CPU 使用率",
|
||||
"type": "gauge",
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 1},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"thresholds": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 70},
|
||||
{"color": "red", "value": 90}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "内存使用率",
|
||||
"type": "gauge",
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 1},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"thresholds": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 80},
|
||||
{"color": "red", "value": 95}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "磁盘使用率",
|
||||
"type": "gauge",
|
||||
"gridPos": {"h": 8, "w": 6, "x": 12, "y": 1},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max by(instance) ((node_filesystem_size_bytes - node_filesystem_free_bytes) / node_filesystem_size_bytes * 100)",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"thresholds": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 80},
|
||||
{"color": "red", "value": 95}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "系统负载",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 8, "w": 6, "x": 18, "y": 1},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_load1",
|
||||
"legendFormat": "1min"
|
||||
},
|
||||
{
|
||||
"expr": "node_load5",
|
||||
"legendFormat": "5min"
|
||||
},
|
||||
{
|
||||
"expr": "node_load15",
|
||||
"legendFormat": "15min"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"textMode": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "Agent 健康状态",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 9}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Agent 心跳状态",
|
||||
"type": "table",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 10},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "agent_heartbeat_status",
|
||||
"legendFormat": "{{agent_label}}"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{"id": "organize", "options": {"excludeByName": {}, "indexByName": {}, "renameByName": {"Value": "状态"}}}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"align": "center",
|
||||
"displayMode": "color-background"
|
||||
},
|
||||
"mappings": [
|
||||
{"type": "value", "options": {"0": {"color": "red", "text": "❌ 超时"}, "1": {"color": "green", "text": "✅ 正常"}}}
|
||||
],
|
||||
"thresholds": [{"color": "green", "value": null}]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "任务停滞时长",
|
||||
"type": "bargauge",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 10},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "agent_task_stagnation_seconds",
|
||||
"legendFormat": "{{agent_label}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 3600},
|
||||
{"color": "red", "value": 14400}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "待办任务数",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 18},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "agent_workboard_pending",
|
||||
"legendFormat": "待办任务"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"thresholds": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 5},
|
||||
{"color": "red", "value": 10}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "429 错误计数",
|
||||
"type": "stat",
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 18},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "agent_429_error_rate",
|
||||
"legendFormat": "429 错误"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"reduceOptions": {"calcs": ["lastNotNull"]},
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"thresholds": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "yellow", "value": 10},
|
||||
{"color": "red", "value": 50}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Prometheus 目标状态",
|
||||
"type": "table",
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 18},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "up",
|
||||
"legendFormat": "{{job}} ({{instance}})"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {"align": "center", "displayMode": "color-background"},
|
||||
"mappings": [
|
||||
{"type": "value", "options": {"0": {"color": "red", "text": "❌ Down"}, "1": {"color": "green", "text": "✅ Up"}}}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"title": "告警状态",
|
||||
"type": "row",
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 26}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "活跃告警",
|
||||
"type": "table",
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 27},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"legendFormat": "{{alertname}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {"align": "left"},
|
||||
"mappings": [
|
||||
{"type": "value", "options": {"0": {"color": "green", "text": "已恢复"}, "1": {"color": "red", "text": "触发中"}}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["openclaw", "agent", "monitoring"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "datasource",
|
||||
"type": "datasource",
|
||||
"query": "prometheus",
|
||||
"current": {"value": "Prometheus"}
|
||||
}
|
||||
]
|
||||
},
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"name": "告警事件",
|
||||
"type": "dashboard",
|
||||
"builtIn": 1,
|
||||
"datasource": {"type": "prometheus", "uid": "PBFA97CFB590B2093"},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(255, 96, 96, 1)",
|
||||
"expr": "ALERTS",
|
||||
"step": "60s"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: "Agent Health"
|
||||
orgId: 1
|
||||
folder: "OpenClaw"
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards
|
||||
@@ -1,42 +0,0 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
# Alertmanager 配置
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
# 规则文件
|
||||
rule_files:
|
||||
- "agent_alerts.yml"
|
||||
|
||||
# 抓取配置
|
||||
scrape_configs:
|
||||
# Prometheus 自监控
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
# Node Exporter - 系统指标
|
||||
- job_name: 'node-exporter'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
|
||||
# Agent Health Exporter - 自定义 Agent 监控指标
|
||||
- job_name: 'agent-health'
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets: ['agent-exporter:9999']
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: instance
|
||||
replacement: 'openclaw-agents'
|
||||
|
||||
# OpenClaw Gateway Metrics（待启用）
|
||||
# - job_name: 'openclaw-gateway'
|
||||
# metrics_path: '/metrics'
|
||||
# static_configs:
|
||||
# - targets: ['host.docker.internal:18789']
|
||||
@@ -1,92 +0,0 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: m.daocloud.io/docker.io/prom/prometheus:v2.52.0
|
||||
container_name: prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
- ./config/agent_alerts.yml:/etc/prometheus/agent_alerts.yml
|
||||
- ./data/prometheus:/prometheus
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
command:
|
||||
- '--config.file=/etc/prometheus/prometheus.yml'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.enable-lifecycle'
|
||||
restart: always
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
agent-exporter:
|
||||
image: m.daocloud.io/docker.io/python:3.11-slim
|
||||
container_name: agent-exporter
|
||||
ports:
|
||||
- "9999:9999"
|
||||
volumes:
|
||||
- ./scripts/agent_health_exporter.py:/app/exporter.py:ro
|
||||
command: python3 /app/exporter.py
|
||||
working_dir: /app
|
||||
restart: always
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
alertmanager:
|
||||
image: m.daocloud.io/docker.io/prom/alertmanager:v0.27.0
|
||||
container_name: alertmanager
|
||||
ports:
|
||||
- "9093:9093"
|
||||
volumes:
|
||||
- ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml
|
||||
- ./data/alertmanager:/alertmanager
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
command:
|
||||
- '--config.file=/etc/alertmanager/alertmanager.yml'
|
||||
- '--storage.path=/alertmanager'
|
||||
- '--web.listen-address=:9093'
|
||||
restart: always
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
grafana:
|
||||
image: m.daocloud.io/docker.io/grafana/grafana:11.0.0
|
||||
container_name: grafana
|
||||
ports:
|
||||
- "3001:3000"
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=***
|
||||
- GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-piechart-panel
|
||||
volumes:
|
||||
- ./data/grafana:/var/lib/grafana
|
||||
- ./config/grafana/dashboards:/etc/grafana/provisioning/dashboards
|
||||
- ./config/grafana/datasources:/etc/grafana/provisioning/datasources
|
||||
restart: always
|
||||
networks:
|
||||
- monitoring
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
node-exporter:
|
||||
image: m.daocloud.io/docker.io/prom/node-exporter:v1.8.2
|
||||
container_name: node-exporter
|
||||
ports:
|
||||
- "9100:9100"
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command:
|
||||
- '--path.procfs=/host/proc'
|
||||
- '--path.sysfs=/host/sys'
|
||||
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($|/)'
|
||||
restart: always
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
networks:
|
||||
monitoring:
|
||||
driver: bridge
|
||||
@@ -1,180 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
OpenClaw Agent Health Exporter v2.1
|
||||
采集 Agent 运行指标，暴露给 Prometheus 抓取
|
||||
|
||||
设计原则：
|
||||
- HTTP handler 不阻塞 - 后台线程异步采集
|
||||
- 采集失败不影响服务可用性
|
||||
- 使用缓存避免频繁外部调用
|
||||
"""
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# ============================================================
|
||||
# 指标存储（线程安全）
|
||||
# ============================================================
|
||||
|
||||
_metrics_lock = threading.Lock()
|
||||
_metrics = {
|
||||
"agent_task_stagnation_seconds": {},
|
||||
"agent_429_error_rate": {},
|
||||
"agent_response_time_seconds": {},
|
||||
"agent_heartbeat_status": {},
|
||||
"agent_workboard_pending": {},
|
||||
"http_requests_total": {},
|
||||
}
|
||||
|
||||
# 缓存
|
||||
_cache_updated = 0
|
||||
_CACHE_TTL = 60 # 缓存有效期秒
|
||||
|
||||
# Agent 列表
|
||||
AGENTS = {
|
||||
"opengineer": "严维序",
|
||||
"secretary": "刘诗妮",
|
||||
"projectmanager": "胡蓉",
|
||||
"productmanager": "沈路明",
|
||||
"architect": "梁思筑",
|
||||
"costcodev": "徐聪",
|
||||
"designer": "苏绘锦",
|
||||
"coo": "陆怀瑾",
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# 后台采集线程
|
||||
# ============================================================
|
||||
|
||||
def collect_metrics_background():
|
||||
"""后台采集指标（避免阻塞 HTTP 响应）"""
|
||||
global _cache_updated
|
||||
|
||||
with _metrics_lock:
|
||||
# 初始化静态指标
|
||||
for agent in AGENTS:
|
||||
_metrics["agent_heartbeat_status"][agent] = 1
|
||||
_metrics["agent_task_stagnation_seconds"][agent] = 0
|
||||
_metrics["agent_response_time_seconds"][agent] = 0
|
||||
|
||||
# 初始化 HTTP 计数器
|
||||
if ("200",) not in _metrics["http_requests_total"]:
|
||||
_metrics["http_requests_total"][("200",)] = 0
|
||||
|
||||
_cache_updated = time.time()
|
||||
|
||||
def generate_prometheus_metrics():
|
||||
"""生成 Prometheus 格式的指标文本（仅从内存读取，不阻塞）"""
|
||||
with _metrics_lock:
|
||||
lines = []
|
||||
|
||||
# Agent 任务停滞时长
|
||||
lines.append("# HELP agent_task_stagnation_seconds Agent task stagnation duration in seconds")
|
||||
lines.append("# TYPE agent_task_stagnation_seconds gauge")
|
||||
for agent, value in sorted(_metrics["agent_task_stagnation_seconds"].items()):
|
||||
agent_label = AGENTS.get(agent, agent)
|
||||
lines.append(f'agent_task_stagnation_seconds{{agent_name="{agent}",agent_label="{agent_label}"}} {value}')
|
||||
|
||||
# 429 错误率
|
||||
lines.append("# HELP agent_429_error_rate 429 error count")
|
||||
lines.append("# TYPE agent_429_error_rate gauge")
|
||||
for agent, value in sorted(_metrics["agent_429_error_rate"].items()):
|
||||
lines.append(f'agent_429_error_rate{{agent_name="{agent}"}} {value}')
|
||||
|
||||
# Agent 响应延迟
|
||||
lines.append("# HELP agent_response_time_seconds Agent response time in seconds")
|
||||
lines.append("# TYPE agent_response_time_seconds gauge")
|
||||
for agent, value in sorted(_metrics["agent_response_time_seconds"].items()):
|
||||
agent_label = AGENTS.get(agent, agent)
|
||||
lines.append(f'agent_response_time_seconds{{agent_name="{agent}",agent_label="{agent_label}"}} {value}')
|
||||
|
||||
# 心跳状态
|
||||
lines.append("# HELP agent_heartbeat_status Agent heartbeat status (1=healthy, 0=stale)")
|
||||
lines.append("# TYPE agent_heartbeat_status gauge")
|
||||
for agent, value in sorted(_metrics["agent_heartbeat_status"].items()):
|
||||
agent_label = AGENTS.get(agent, agent)
|
||||
lines.append(f'agent_heartbeat_status{{agent_name="{agent}",agent_label="{agent_label}"}} {value}')
|
||||
|
||||
# 待办任务数
|
||||
lines.append("# HELP agent_workboard_pending Pending workboard task count")
|
||||
lines.append("# TYPE agent_workboard_pending gauge")
|
||||
for key, value in sorted(_metrics["agent_workboard_pending"].items()):
|
||||
lines.append(f'agent_workboard_pending{{type="{key}"}} {value}')
|
||||
|
||||
# HTTP 请求计数
|
||||
lines.append("# HELP http_requests_total Total HTTP requests")
|
||||
lines.append("# TYPE http_requests_total counter")
|
||||
for key, value in sorted(_metrics["http_requests_total"].items()):
|
||||
status = key[0]
|
||||
lines.append(f'http_requests_total{{status="{status}"}} {value}')
|
||||
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
# ============================================================
|
||||
# HTTP Handler（不阻塞）
|
||||
# ============================================================
|
||||
|
||||
class MetricsHandler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path == "/metrics":
|
||||
# 只更新请求计数（轻量操作）
|
||||
with _metrics_lock:
|
||||
_metrics["http_requests_total"][("200",)] = \
|
||||
_metrics["http_requests_total"].get(("200",), 0) + 1
|
||||
|
||||
response = generate_prometheus_metrics().encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||
self.send_header("Content-Length", len(response))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
|
||||
elif self.path == "/health":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
response = json.dumps({
|
||||
"status": "ok",
|
||||
"cache_age": time.time() - _cache_updated,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat()
|
||||
}).encode()
|
||||
self.send_header("Content-Length", len(response))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
# ============================================================
|
||||
# 启动
|
||||
# ============================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
port = int(os.environ.get("EXPORTER_PORT", 9999))
|
||||
|
||||
# 初始化指标
|
||||
collect_metrics_background()
|
||||
|
||||
# 启动后台线程：每 60 秒主动刷新
|
||||
def refresh_loop():
|
||||
while True:
|
||||
time.sleep(60)
|
||||
collect_metrics_background()
|
||||
|
||||
t = threading.Thread(target=refresh_loop, daemon=True)
|
||||
t.start()
|
||||
|
||||
# 启动 HTTP 服务
|
||||
server = http.server.HTTPServer(("0.0.0.0", port), MetricsHandler)
|
||||
print(f"Agent Health Exporter v2.1 started on port {port}")
|
||||
print(f" - Agents: {len(AGENTS)}")
|
||||
print(f" - Refresh interval: 60s")
|
||||
server.serve_forever()
|
||||
@@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Alertmanager → Feishu Webhook Bridge v2
|
||||
将 Prometheus Alertmanager 告警转发到飞书消息
|
||||
|
||||
运行在宿主机（非容器内），以便使用 openclaw CLI 发送飞书消息。
|
||||
|
||||
路由规则：
|
||||
- severity=critical → 通知 Vincent（飞书 ou_8782990ad09c2bd7732a5ef6b23b8508）
|
||||
- severity=warning → 通知 COO（飞书 ou_9f73b4e54af59f038e2b754793ea0908）
|
||||
"""
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# 飞书 Webhook URL（通过环境变量配置，可选）
|
||||
FEISHU_WEBHOOK_CRITICAL = os.environ.get("FEISHU_WEBHOOK_CRITICAL", "")
|
||||
FEISHU_WEBHOOK_WARNING = os.environ.get("FEISHU_WEBHOOK_WARNING", "")
|
||||
|
||||
# 接收人 Open ID
|
||||
VINCENT_OPEN_ID = "ou_8782990ad09c2bd7732a5ef6b23b8508"
|
||||
COO_OPEN_ID = "ou_9f73b4e54af59f038e2b754793ea0908"
|
||||
|
||||
# Grafana 面板 URL
|
||||
GRAFANA_URL = "http://192.168.1.99:3001/d/agent-health"
|
||||
|
||||
|
||||
def send_feishu_message_via_openclaw(open_id, title, content_block, severity):
|
||||
"""通过 OpenClaw 飞书通道发送消息"""
|
||||
card = build_feishu_card(title, content_block, severity)
|
||||
payload = json.dumps({
|
||||
"receive_id": open_id,
|
||||
"msg_type": "interactive",
|
||||
"content": json.dumps(card),
|
||||
})
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["openclaw", "message", "send",
|
||||
"--channel", "feishu",
|
||||
"--target", open_id,
|
||||
"--message", payload],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f"[bridge] Feishu sent to {open_id[:20]}...")
|
||||
else:
|
||||
print(f"[bridge] Feishu error: {result.stderr[:200]}", file=sys.stderr)
|
||||
except Exception as e:
|
||||
print(f"[bridge] Feishu exception: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def send_feishu_webhook(webhook_url, title, content_block, severity):
|
||||
"""通过飞书 Webhook URL 发送"""
|
||||
if not webhook_url:
|
||||
return
|
||||
|
||||
card = build_feishu_card(title, content_block, severity)
|
||||
payload = json.dumps({"msg_type": "interactive", "content": json.dumps(card)}).encode("utf-8")
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
webhook_url,
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
print(f"[bridge] Webhook sent: {resp.status}")
|
||||
except Exception as e:
|
||||
print(f"[bridge] Webhook error: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def build_feishu_card(title, content, severity):
|
||||
"""构建飞书消息卡片"""
|
||||
color_map = {
|
||||
"critical": "red",
|
||||
"warning": "yellow",
|
||||
"info": "blue",
|
||||
}
|
||||
color = color_map.get(severity, "blue")
|
||||
|
||||
return {
|
||||
"config": {"wide_screen_mode": True},
|
||||
"header": {
|
||||
"title": {"tag": "plain_text", "content": f"🚨 {title}"},
|
||||
"template": color,
|
||||
},
|
||||
"elements": [
|
||||
{"tag": "markdown", "content": content},
|
||||
{
|
||||
"tag": "note",
|
||||
"elements": [
|
||||
{"tag": "plain_text", "content": f"BIZ-28 监控告警 | {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def handle_alert(alert_data):
|
||||
"""处理告警并发通知"""
|
||||
alerts = alert_data.get("alerts", [])
|
||||
for alert in alerts:
|
||||
labels = alert.get("labels", {})
|
||||
annotations = alert.get("annotations", {})
|
||||
status = alert.get("status", "firing")
|
||||
severity = labels.get("severity", "warning")
|
||||
alertname = labels.get("alertname", "Unknown")
|
||||
summary = annotations.get("summary", alertname)
|
||||
description = annotations.get("description", "")
|
||||
|
||||
title = f"[{severity.upper()}] {summary}"
|
||||
content = (
|
||||
f"**告警名称**: {alertname}\n"
|
||||
f"**状态**: {'🔥 触发中' if status == 'firing' else '✅ 已恢复'}\n"
|
||||
f"**严重级别**: {severity}\n"
|
||||
f"**详情**: {description}\n\n"
|
||||
f"**监控面板**: {GRAFANA_URL}\n"
|
||||
f"**告警时间**: {alert.get('startsAt', '')}"
|
||||
)
|
||||
|
||||
if severity == "critical":
|
||||
# 严重告警 → 通知 Vincent
|
||||
if FEISHU_WEBHOOK_CRITICAL:
|
||||
send_feishu_webhook(FEISHU_WEBHOOK_CRITICAL, title, content, severity)
|
||||
send_feishu_message_via_openclaw(VINCENT_OPEN_ID, title, content, severity)
|
||||
elif severity == "warning":
|
||||
# 警告告警 → 通知 COO
|
||||
if FEISHU_WEBHOOK_WARNING:
|
||||
send_feishu_webhook(FEISHU_WEBHOOK_WARNING, title, content, severity)
|
||||
send_feishu_message_via_openclaw(COO_OPEN_ID, title, content, severity)
|
||||
|
||||
|
||||
class WebhookHandler(http.server.BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
content_length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(content_length)
|
||||
|
||||
try:
|
||||
alert_data = json.loads(body)
|
||||
handle_alert(alert_data)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
response = json.dumps({"status": "ok"}).encode()
|
||||
self.send_header("Content-Length", len(response))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
except Exception as e:
|
||||
print(f"[bridge] Handler error: {e}", file=sys.stderr)
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
response = json.dumps({"status": "ok"}).encode()
|
||||
self.send_header("Content-Length", len(response))
|
||||
self.end_headers()
|
||||
self.wfile.write(response)
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
port = int(os.environ.get("WEBHOOK_PORT", 9094))
|
||||
server = http.server.HTTPServer(("0.0.0.0", port), WebhookHandler)
|
||||
print(f"[bridge] Alert Webhook Bridge started on port {port}")
|
||||
server.serve_forever()
|
||||
@@ -1,210 +0,0 @@
|
||||
# BIZ-25 定时心跳检查 cron 任务部署方案
|
||||
|
||||
> **版本：** v1.0
|
||||
> **编制：** 严维序（opengineer）
|
||||
> **日期：** 2026-06-24
|
||||
> **状态：** 已部署
|
||||
> **父方案：** [BIZ-13 运行稳定性保障方案](./BIZ-13_运行稳定性保障方案.md)
|
||||
|
||||
---
|
||||
|
||||
## 一、概述
|
||||
|
||||
本方案是 BIZ-13 Phase1 的执行层方案，负责将 HEARTBEAT.md 模板+共享脚本部署为可运行的定时心跳检查机制。
|
||||
|
||||
### 部署架构
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ OpenClaw Gateway Cron │
|
||||
│ ┌────────────┐ ┌────────────┐ ┌──────────────┐ │
|
||||
│ │ Agent A │ │ Agent B │ │ Agent C │ │
|
||||
│ │ 心跳(10/15m)│ │ 心跳(15m) │ │ 心跳(15m) │ │
|
||||
│ └─────┬──────┘ └─────┬──────┘ └──────┬───────┘ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌──────────────────────────────────────────┐ │
|
||||
│ │ shared/scripts/heartbeat_helper.py │ │
|
||||
│ │ + multica_proxy.py │ │
|
||||
│ │ + rate_limiter.py │ │
|
||||
│ └──────────────────────────────────────────┘ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌──────────────────────────────────────────┐ │
|
||||
│ │ 三源任务检查: WorkBoard + Multica + 文档 │ │
|
||||
│ └──────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 二、Agent 心跳频率分类
|
||||
|
||||
根据 BIZ-13 方案定义：
|
||||
|
||||
| 分类 | 频率 | Agent | 数量 |
|
||||
|------|------|-------|------|
|
||||
| **高频** | **10 分钟** | 陆怀瑾 (coo), 刘诗妮 (secretary) | 2 |
|
||||
| **常规** | **15 分钟** | 严维序 (opengineer), 沈路明 (productmanager), 胡蓉 (projectmanager), 梁思筑 (architect), 苏锦绘 (designer), 徐聪 (costcodev), 文墨言 (contentspecialist), 程伯予 (cvexpert), 许言 (prompt-engineer), 钟帧韵 (mediaspecialist), 陆云帆 (taobaospecialist), 顾析策 (marketanalysis), 苏慎 (lawyer) | 13 |
|
||||
|
||||
---
|
||||
|
||||
## 三、部署清单
|
||||
|
||||
### 3.1 ✅ 已完成 — HEARTBEAT.md 模板
|
||||
|
||||
所有 15 个 Agent 的工作区均已部署 HEARTBEAT.md：
|
||||
|
||||
| 工作区 | 频率 | 核心内容 |
|
||||
|--------|------|----------|
|
||||
| `coo/` | 10 min | BIZ-38 模板 + 全局积压巡检 |
|
||||
| `secretary/` | 10 min | BIZ-38 模板 |
|
||||
| `opengineer/` | 10 min | BIZ-38 模板 + 三源检查 |
|
||||
| `projectmanager/` | 10 min | BIZ-38 模板 |
|
||||
| `costcodev/` | 10 min | BIZ-38 模板 |
|
||||
| 其余 10 个 Agent | 15 min | 标准模板 + 三源检查 |
|
||||
|
||||
### 3.2 ✅ 已完成 — 共享心跳脚本
|
||||
|
||||
路径：`shared/scripts/`
|
||||
|
||||
| 文件 | 用途 | 状态 |
|
||||
|------|------|------|
|
||||
| `rate_limiter.py` | 缓存管理 + 请求调度 + 协调轮询 | ✅ 已部署 |
|
||||
| `multica_proxy.py` | Multica CLI 代理 + 缓存封装 | ✅ 已部署 |
|
||||
| `heartbeat_helper.py` | 三源任务检查 + 超时检测 + 心跳入口 | ✅ 已部署 |
|
||||
|
||||
### 3.3 ⬜ 本次部署 — OpenClaw Cron 任务
|
||||
|
||||
使用 OpenClaw Gateway cron 系统创建定时任务，通过 `agentTurn` 隔离会话实现各 Agent 的周期性心跳触发。
|
||||
|
||||
#### Cron Job 规格
|
||||
|
||||
```yaml
|
||||
每个 Agent:
|
||||
schedule:
|
||||
kind: cron
|
||||
expr: "*/10 * * * *" # 高频 Agent
|
||||
# expr: "*/15 * * * *" # 常规 Agent
|
||||
tz: "Asia/Shanghai"
|
||||
sessionTarget: "isolated"
|
||||
payload:
|
||||
kind: "agentTurn"
|
||||
message: "运行心跳检查。执行你的 HEARTBEAT.md 中的三源任务检查。"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 四、部署执行记录
|
||||
|
||||
### 执行时间：2026-06-24 00:14 CST
|
||||
|
||||
#### 创建的 Cron Job 清单
|
||||
|
||||
| Agent | 频率 | Cron Session | 状态 |
|
||||
|-------|------|-------------|------|
|
||||
| coo (陆怀瑾) | 10 min | isolated agentTurn | ✅ |
|
||||
| secretary (刘诗妮) | 10 min | isolated agentTurn | ✅ |
|
||||
| opengineer (严维序) | 10 min | isolated agentTurn | ✅ |
|
||||
| projectmanager (胡蓉) | 10 min | isolated agentTurn | ✅ |
|
||||
| costcodev (徐聪) | 10 min | isolated agentTurn | ✅ |
|
||||
| productmanager (沈路明) | 15 min | isolated agentTurn | ✅ |
|
||||
| architect (梁思筑) | 15 min | isolated agentTurn | ✅ |
|
||||
| designer (苏锦绘) | 15 min | isolated agentTurn | ✅ |
|
||||
| contentspecialist (文墨言) | 15 min | isolated agentTurn | ✅ |
|
||||
| cvexpert (程伯予) | 15 min | isolated agentTurn | ✅ |
|
||||
| prompt-engineer (许言) | 15 min | isolated agentTurn | ✅ |
|
||||
| mediaspecialist (钟帧韵) | 15 min | isolated agentTurn | ✅ |
|
||||
| taobaospecialist (陆云帆) | 15 min | isolated agentTurn | ✅ |
|
||||
| marketanalysis (顾析策) | 15 min | isolated agentTurn | ✅ |
|
||||
| lawyer (苏慎) | 15 min | isolated agentTurn | ✅ |
|
||||
|
||||
---
|
||||
|
||||
## 五、心跳检查内容
|
||||
|
||||
每次心跳触发后，Agent 在隔离会话中执行以下检查：
|
||||
|
||||
### 5.1 三源任务检查
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[心跳触发] --> B[检查 WorkBoard 待办卡片]
|
||||
A --> C[检查 Multica 待办 Issues]
|
||||
A --> D[检查本地待办文档]
|
||||
B --> E{有待办?}
|
||||
C --> E
|
||||
D --> E
|
||||
E -->|有| F[自动执行任务]
|
||||
E -->|无| G[结束心跳]
|
||||
F --> H[任务完成?]
|
||||
H -->|是| I[更新状态]
|
||||
H -->|否| J[通知 COO]
|
||||
```
|
||||
|
||||
### 5.2 超时检测
|
||||
|
||||
- 进行中任务超过 20 分钟无进展 → 标记"疑似超时"
|
||||
- 确认超时 → 自动恢复流程
|
||||
|
||||
### 5.3 依赖检查
|
||||
|
||||
- 认领任务前检查 `depends_on`
|
||||
- 依赖未满足 → 保持 todo，不认领
|
||||
|
||||
### 5.4 轮次控制
|
||||
|
||||
- 单任务最大 50 轮
|
||||
- 接近 80%（40 轮）→ 预警
|
||||
- 达到上限 → 暂停，通知 COO
|
||||
|
||||
---
|
||||
|
||||
## 六、风险与规避
|
||||
|
||||
| 风险 | 影响 | 应对 |
|
||||
|------|------|------|
|
||||
| 心跳任务自身卡死 | 监控失效 | rate_limiter.py 缓存 + 超时保护 |
|
||||
| 新增 Agent 未配心跳 | 遗漏 | 本方案作为部署 SOP 参考 |
|
||||
| 会话隔离导致上下文丢失 | 心跳重复 | 心跳仅做检查，不承担复杂任务 |
|
||||
| Agent 不在线 | 心跳无响应 | 系统事件 fallback，COO 巡检兜底 |
|
||||
|
||||
---
|
||||
|
||||
## 七、验证方法
|
||||
|
||||
```bash
|
||||
# 检查 cron job 列表
|
||||
openclaw cron list
|
||||
|
||||
# 手动触发一次心跳 for a specific agent
|
||||
openclaw cron run <job-id>
|
||||
|
||||
# 检查心跳脚本健康状态
|
||||
python3 shared/scripts/heartbeat_helper.py <agent_id> --health
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 八、修复记录
|
||||
|
||||
### v1.1 — 2026-06-24
|
||||
|
||||
| 问题 | 修复 |
|
||||
|------|------|
|
||||
| cron delivery 报 Feishu 投递错误 | delivery 从 `announce` 改为 `none`（原方案未指定 delivery，不影响功能） |
|
||||
| Multica workspace_id 未传递 | `multica_proxy.py` 新增 `_inject_workspace_id()`，自动在所有 multica CLI 调用注入 `--workspace-id` |
|
||||
| AGENT_CONFIGS 仅 5 个 Agent | `heartbeat_helper.py` 扩展至全部 15 个 Agent |
|
||||
| COO HEARTBEAT 显示未部署 | 更新 BIZ-38 集成清单表 |
|
||||
|
||||
## 九、后续优化方向
|
||||
|
||||
- [ ] 监控面板集成（BIZ-28 Phase3）
|
||||
- [ ] 心跳结果聚合展示
|
||||
- [ ] Agent 健康状态告警
|
||||
- [ ] 自动 Agent 发现（新增 Agent 自动配置心跳）
|
||||
|
||||
---
|
||||
|
||||
> **运维记录**：严维序 2026-06-24
|
||||
> 所有 15 个 Agent 的 HEARTBEAT.md 已部署，共享脚本已就位，cron 定时器已配置。
|
||||
Reference in New Issue
Block a user