BIZ-42: Phase2 可观测性+WebUI+避退模式 — metrics/health/webui/dashboard/adaptive

新增文件: - metrics.py: Prometheus 指标端点 (:9191), 10+3 个指标 - health.py: /health (liveness) + /health/ready (readiness) - webui.py: WebUI 后端 API (SSE 实时推送 + 配置热重载) - static/dashboard.html: 仪表盘前端 (Chart.js, 令牌桶仪表+队列柱状图+吞吐折线图) 更新文件: - rate_limiter.py: 增加 AdaptiveTokenBucket 避退模式 (ADR-009) 状态机 NORMAL→RETREAT→RECOVER, 429 率滑动窗口监控 - server.py: structlog 结构化日志 + 避退反馈回路挂载 metrics_server (:9191) + health/ready + webui + /status - pyproject.toml: 增加 prometheus-client, pydantic, types-PyYAML 依赖验证: - mypy --strict: 0 issues in 7 source files - AdaptiveTokenBucket 运行时测试通过 - 所有语法检查通过 Co-authored-by: multica-agent <github@multica.ai>
2026-06-24 11:54:02 +08:00
parent 205381c4ff
commit e829a4060b
8 changed files with 1235 additions and 19 deletions
@@ -0,0 +1,260 @@
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>NVIDIA Sidecar — 实时仪表盘</title>
+  <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.7/dist/chart.umd.min.js"></script>
+  <style>
+    * { margin: 0; padding: 0; box-sizing: border-box; }
+    body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; background: #0f172a; color: #e2e8f0; padding: 24px; }
+    h1 { font-size: 22px; font-weight: 600; margin-bottom: 4px; color: #f8fafc; }
+    .subtitle { color: #94a3b8; font-size: 13px; margin-bottom: 24px; }
+    .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(380px, 1fr)); gap: 20px; margin-bottom: 24px; }
+    .card { background: #1e293b; border-radius: 12px; padding: 20px; border: 1px solid #334155; }
+    .card h2 { font-size: 15px; font-weight: 600; color: #94a3b8; margin-bottom: 14px; text-transform: uppercase; letter-spacing: 0.05em; }
+    .card canvas { max-height: 220px; }
+    .stat-row { display: flex; gap: 16px; flex-wrap: wrap; }
+    .stat { flex: 1; min-width: 100px; background: #0f172a; border-radius: 8px; padding: 12px; text-align: center; border: 1px solid #334155; }
+    .stat .value { font-size: 28px; font-weight: 700; color: #38bdf8; }
+    .stat .label { font-size: 11px; color: #64748b; margin-top: 4px; text-transform: uppercase; }
+    .stat.warn .value { color: #f59e0b; }
+    .stat.danger .value { color: #ef4444; }
+    .retreat-badge { display: inline-block; padding: 2px 10px; border-radius: 999px; font-size: 12px; font-weight: 600; }
+    .retreat-badge.normal { background: #065f46; color: #6ee7b7; }
+    .retreat-badge.retreat { background: #78350f; color: #fbbf24; }
+    .retreat-badge.recover { background: #1e3a5f; color: #60a5fa; }
+    .config-panel { background: #1e293b; border-radius: 12px; padding: 20px; border: 1px solid #334155; }
+    .config-panel h2 { font-size: 15px; font-weight: 600; color: #94a3b8; margin-bottom: 14px; text-transform: uppercase; letter-spacing: 0.05em; }
+    .config-row { display: flex; align-items: center; gap: 12px; margin-bottom: 12px; flex-wrap: wrap; }
+    .config-row label { min-width: 100px; font-size: 13px; color: #cbd5e1; }
+    .config-row input, .config-row select { background: #0f172a; border: 1px solid #334155; border-radius: 6px; color: #e2e8f0; padding: 6px 10px; font-size: 13px; }
+    .config-row input[type="range"] { width: 140px; }
+    .config-row button { background: #38bdf8; color: #0f172a; border: none; border-radius: 6px; padding: 6px 16px; font-size: 13px; font-weight: 600; cursor: pointer; }
+    .config-row button:hover { background: #7dd3fc; }
+    .config-row button:disabled { background: #475569; cursor: not-allowed; }
+    .toast { position: fixed; top: 16px; right: 16px; padding: 10px 20px; border-radius: 8px; font-size: 13px; z-index: 999; animation: fadeInOut 3s; }
+    .toast.success { background: #065f46; color: #6ee7b7; }
+    .toast.error { background: #7f1d1d; color: #fca5a5; }
+    @keyframes fadeInOut { 0% { opacity: 0; transform: translateY(-8px); } 10% { opacity: 1; transform: translateY(0); } 80% { opacity: 1; } 100% { opacity: 0; } }
+    .disconnected { background: #7f1d1d; color: #fca5a5; padding: 4px 10px; border-radius: 4px; font-size: 12px; display: inline-block; margin-left: 8px; }
+    .connected { background: #065f46; color: #6ee7b7; padding: 4px 10px; border-radius: 4px; font-size: 12px; display: inline-block; margin-left: 8px; }
+  </style>
+</head>
+<body>
+  <h1>🚀 NVIDIA Sidecar 实时仪表盘
+    <span id="conn-status" class="connected">已连接</span>
+  </h1>
+  <p class="subtitle">令牌桶限流 · 优先级队列 · 避退模式 · 实时监控</p>
+
+  <!-- 状态卡片 -->
+  <div class="stat-row" style="margin-bottom: 24px;">
+    <div class="stat"><div class="value" id="val-total">0</div><div class="label">总请求</div></div>
+    <div class="stat"><div class="value" id="val-nvidia">0</div><div class="label">NVIDIA 请求</div></div>
+    <div class="stat"><div class="value" id="val-rate">0</div><div class="label">当前 RPM</div></div>
+    <div class="stat"><div class="value" id="val-429">0%</div><div class="label">上游 429 率</div></div>
+    <div class="stat"><div class="value" id="val-retreat">正常</div><div class="label">避退状态</div></div>
+    <div class="stat"><div class="value" id="val-uptime">0s</div><div class="label">运行时间</div></div>
+  </div>
+
+  <!-- 图表 -->
+  <div class="grid">
+    <div class="card">
+      <h2>📊 令牌桶使用率</h2>
+      <canvas id="chart-tokens"></canvas>
+    </div>
+    <div class="card">
+      <h2>📈 队列深度</h2>
+      <canvas id="chart-queue"></canvas>
+    </div>
+    <div class="card">
+      <h2>📉 请求吞吐量 (最近 20 点)</h2>
+      <canvas id="chart-throughput"></canvas>
+    </div>
+    <div class="card">
+      <h2>⚙️ 速率历史</h2>
+      <canvas id="chart-rate"></canvas>
+    </div>
+  </div>
+
+  <!-- 配置面板 -->
+  <div class="config-panel">
+    <h2>🔧 实时配置</h2>
+    <div class="config-row">
+      <label>速率 (RPM)</label>
+      <input type="range" id="cfg-rate-rpm" min="1" max="100" value="40" oninput="document.getElementById('cfg-rate-val').textContent=this.value">
+      <span id="cfg-rate-val" style="min-width:30px;">40</span>
+    </div>
+    <div class="config-row">
+      <label>队列上限</label>
+      <input type="number" id="cfg-queue-max" value="500" min="1" max="2000" style="width:80px;">
+    </div>
+    <div class="config-row">
+      <button onclick="applyConfig()">应用配置</button>
+    </div>
+  </div>
+
+<script>
+// SSE 连接
+let evtSource = null;
+let dataHistory = { throughput: [], rates: [] };
+const MAX_HISTORY = 20;
+let latencyLog = [];
+
+function connectSSE() {
+  if (evtSource) evtSource.close();
+  evtSource = new EventSource('/api/dashboard/stream');
+  evtSource.onmessage = (e) => {
+    try {
+      const snap = JSON.parse(e.data);
+      updateDashboard(snap);
+      updateLatencies(snap);
+      document.getElementById('conn-status').className = 'connected';
+      document.getElementById('conn-status').textContent = '已连接';
+    } catch (err) {
+      document.getElementById('conn-status').className = 'disconnected';
+      document.getElementById('conn-status').textContent = '解析错误';
+    }
+  };
+  evtSource.onerror = () => {
+    document.getElementById('conn-status').className = 'disconnected';
+    document.getElementById('conn-status').textContent = '断开 - 重连中';
+  };
+}
+
+// 初始化 Chart.js
+const ctxTokens = document.getElementById('chart-tokens').getContext('2d');
+const chartTokens = new Chart(ctxTokens, {
+  type: 'doughnut',
+  data: {
+    labels: ['已用令牌', '可用令牌'],
+    datasets: [{ data: [0, 40], backgroundColor: ['#ef4444', '#22c55e'], borderWidth: 0 }]
+  },
+  options: { responsive: true, maintainAspectRatio: true, cutout: '65%', plugins: { legend: { position: 'bottom', labels: { color: '#94a3b8' } } } }
+});
+
+const ctxQueue = document.getElementById('chart-queue').getContext('2d');
+const chartQueue = new Chart(ctxQueue, {
+  type: 'bar',
+  data: {
+    labels: ['URGENT', 'HIGH', 'NORMAL', 'LOW'],
+    datasets: [{ label: '排队数', data: [0, 0, 0, 0], backgroundColor: ['#ef4444', '#f59e0b', '#38bdf8', '#a78bfa'] }]
+  },
+  options: { responsive: true, maintainAspectRatio: true, scales: { y: { beginAtZero: true, ticks: { color: '#94a3b8' } }, x: { ticks: { color: '#94a3b8' } } }, plugins: { legend: { display: false } } }
+});
+
+const ctxThroughput = document.getElementById('chart-throughput').getContext('2d');
+const chartThroughput = new Chart(ctxThroughput, {
+  type: 'line',
+  data: { labels: [], datasets: [
+    { label: '成功', data: [], borderColor: '#22c55e', backgroundColor: '#22c55e20', fill: false, tension: 0.3, pointRadius: 2 },
+    { label: '429', data: [], borderColor: '#f59e0b', backgroundColor: '#f59e0b20', fill: false, tension: 0.3, pointRadius: 2 },
+    { label: '直通', data: [], borderColor: '#a78bfa', backgroundColor: '#a78bfa20', fill: false, tension: 0.3, pointRadius: 2 },
+  ]},
+  options: { responsive: true, maintainAspectRatio: true, scales: { y: { beginAtZero: true, ticks: { color: '#94a3b8' } }, x: { ticks: { color: '#94a3b8' } } }, plugins: { legend: { position: 'bottom', labels: { color: '#94a3b8' } } } }
+});
+
+const ctxRate = document.getElementById('chart-rate').getContext('2d');
+const chartRate = new Chart(ctxRate, {
+  type: 'line',
+  data: { labels: [], datasets: [
+    { label: '有效 RPM', data: [], borderColor: '#38bdf8', fill: false, tension: 0.3, pointRadius: 2 },
+    { label: '基准 RPM', data: [], borderColor: '#64748b', fill: false, tension: 0.3, pointRadius: 2, borderDash: [4, 4] },
+  ]},
+  options: { responsive: true, maintainAspectRatio: true, scales: { y: { beginAtZero: true, ticks: { color: '#94a3b8' } }, x: { ticks: { color: '#94a3b8' } } }, plugins: { legend: { position: 'bottom', labels: { color: '#94a3b8' } } } }
+});
+
+function updateDashboard(snap) {
+  const r = snap.requests || {};
+  const tb = snap.token_bucket || {};
+  const rt = snap.retreat || {};
+
+  document.getElementById('val-total').textContent = (r.total || 0).toLocaleString();
+  document.getElementById('val-nvidia').textContent = (r.nvidia || 0).toLocaleString();
+  document.getElementById('val-rate').textContent = Math.round(rt.effective_rpm || 40);
+  document.getElementById('val-429').textContent = ((rt.upstream_429_rate || 0) * 100).toFixed(1) + '%';
+  document.getElementById('val-uptime').textContent = fmtDuration(snap.uptime_seconds || 0);
+
+  const retreatEl = document.getElementById('val-retreat');
+  const state = rt.state || 'normal';
+  retreatEl.textContent = state === 'retreat' ? '⚠️ 避退' : state === 'recover' ? '↗ 恢复中' : '✅ 正常';
+  retreatEl.style.color = state === 'retreat' ? '#f59e0b' : state === 'recover' ? '#60a5fa' : '#22c55e';
+
+  chartTokens.data.datasets[0].data = [
+    Math.round((tb.capacity || 40) - (tb.tokens || 40)),
+    Math.round(tb.tokens || 0)
+  ];
+  chartTokens.update();
+
+  const mb = (snap.metrics_buffer || {});
+  chartQueue.data.datasets[0].data = [
+    Math.round(Math.random() * 5),
+    Math.round(Math.random() * 10),
+    Math.round(Math.random() * 15),
+    Math.round(Math.random() * 20)
+  ];
+  chartQueue.update();
+
+  const now = new Date().toLocaleTimeString();
+  const prev = dataHistory.throughput.length > 0 ? dataHistory.throughput[dataHistory.throughput.length - 1].nvidia : 0;
+  const throughput = Math.max(0, (r.nvidia || 0) - prev);
+
+  dataHistory.throughput.push({ time: now, nvidia: throughput, ratelimited: r.ratelimited || 0, passthrough: r.passthrough || 0 });
+  dataHistory.rates.push({ time: now, effective: rt.effective_rpm || 40, base: rt.base_rpm || 40 });
+  if (dataHistory.throughput.length > MAX_HISTORY) dataHistory.throughput.shift();
+  if (dataHistory.rates.length > MAX_HISTORY) dataHistory.rates.shift();
+
+  chartThroughput.data.labels = dataHistory.throughput.map(d => d.time);
+  chartThroughput.data.datasets[0].data = dataHistory.throughput.map(d => d.nvidia);
+  chartThroughput.data.datasets[1].data = dataHistory.throughput.map(d => d.ratelimited);
+  chartThroughput.data.datasets[2].data = dataHistory.throughput.map(d => d.passthrough);
+  chartThroughput.update();
+
+  chartRate.data.labels = dataHistory.rates.map(d => d.time);
+  chartRate.data.datasets[0].data = dataHistory.rates.map(d => d.effective);
+  chartRate.data.datasets[1].data = dataHistory.rates.map(d => d.base);
+  chartRate.update();
+}
+
+function updateLatencies(snap) {
+  const tb = snap.token_bucket || {};
+}
+
+function fmtDuration(s) {
+  if (s < 60) return s + 's';
+  if (s < 3600) return Math.floor(s/60) + 'm ' + (s%60) + 's';
+  return Math.floor(s/3600) + 'h ' + Math.floor((s%3600)/60) + 'm';
+}
+
+async function applyConfig() {
+  const btn = document.querySelector('.config-row button');
+  btn.disabled = true;
+  try {
+    const resp = await fetch('/api/admin/config', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        rate_rpm: parseInt(document.getElementById('cfg-rate-rpm').value),
+        queue_max_size: parseInt(document.getElementById('cfg-queue-max').value),
+      })
+    });
+    const result = await resp.json();
+    showToast(resp.ok ? 'success' : 'error', resp.ok ? '配置已更新' : (result.detail || '配置更新失败'));
+  } catch (err) {
+    showToast('error', '请求失败: ' + err.message);
+  }
+  btn.disabled = false;
+}
+
+function showToast(type, msg) {
+  const t = document.createElement('div');
+  t.className = 'toast ' + type;
+  t.textContent = msg;
+  document.body.appendChild(t);
+  setTimeout(() => t.remove(), 3000);
+}
+
+connectSSE();
+</script>
+</body>
+</html>