feat: Sidecar V2 — multi-pool provider proxy with 429 cooldown
- proxy.py: Fix route path duplication (v1/v1 → v1) when upstream base URL already includes /v1 prefix - proxy.py: Fix _emergency_count global variable for metrics tracking - server.py: Add logging.basicConfig(level=logging.INFO) for structlog INFO-level log visibility - Full multi-pool routing: primary → fallback → emergency passthrough - Per-backend rate limiting with RPM-based token bucket - 429 cooldown mechanism with automatic recovery - Dashboard with SSE real-time monitoring - Admin API for backend/pool/config management - SQLite-backed persistence with encrypted API key storage - Docker compose deployment Deployed by opengineer 严维序 as BIZ-50 Step 4
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
"""Provider pool management: primary / fallback pool routing."""
|
||||
|
||||
import structlog
|
||||
from typing import Optional
|
||||
|
||||
from storage.backend_store import list_backends, get_pool_stats
|
||||
from storage.models import Backend
|
||||
|
||||
logger = structlog.get_logger("sidecar_v2.pool_manager")
|
||||
|
||||
|
||||
class PoolManager:
|
||||
"""Manages provider pools and selects healthy backends for a given model.
|
||||
|
||||
Priority: primary pool → fallback pool.
|
||||
Within a pool: healthy backends only, sorted by availability.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._pool_order = ["primary", "fallback"]
|
||||
|
||||
def get_available_backends(
|
||||
self, canonical_model: str, pool: Optional[str] = None
|
||||
) -> list[Backend]:
|
||||
"""Get all healthy, enabled backends that serve a model, in pool order.
|
||||
|
||||
Args:
|
||||
canonical_model: Canonical model name to match.
|
||||
pool: Optional pool filter (primary/fallback). None = all pools.
|
||||
|
||||
Returns:
|
||||
List of ready backends sorted by pool priority, then RPM utilization.
|
||||
"""
|
||||
backends: list[Backend] = []
|
||||
|
||||
pools_to_check = [pool] if pool else self._pool_order
|
||||
for p in pools_to_check:
|
||||
pool_backends = list_backends(pool=p, enabled_only=True, decrypt_key=True)
|
||||
for b in pool_backends:
|
||||
if b.status == "healthy" and b.has_model(canonical_model):
|
||||
backends.append(b)
|
||||
if pool:
|
||||
break
|
||||
|
||||
return backends
|
||||
|
||||
def get_any_healthy_backends(self, pool: Optional[str] = None) -> list[Backend]:
|
||||
"""Get all healthy, enabled backends regardless of model."""
|
||||
backends: list[Backend] = []
|
||||
pools_to_check = [pool] if pool else self._pool_order
|
||||
for p in pools_to_check:
|
||||
pool_backends = list_backends(pool=p, enabled_only=True, decrypt_key=True)
|
||||
for b in pool_backends:
|
||||
if b.status == "healthy":
|
||||
backends.append(b)
|
||||
if pool:
|
||||
break
|
||||
return backends
|
||||
|
||||
def get_pool_status(self) -> dict:
|
||||
"""Get pool summary for dashboard."""
|
||||
stats = get_pool_stats()
|
||||
result = {}
|
||||
for pool in self._pool_order:
|
||||
s = stats.get(pool, {"total": 0, "enabled": 0, "healthy": 0, "cooling": 0, "error": 0})
|
||||
result[pool] = s
|
||||
# Also include any other pools
|
||||
for pool, s in stats.items():
|
||||
if pool not in result:
|
||||
result[pool] = s
|
||||
return result
|
||||
|
||||
def is_pool_available(self, canonical_model: str, pool: str = "primary") -> bool:
|
||||
"""Check if a pool has any healthy backends for a model."""
|
||||
backends = self.get_available_backends(canonical_model, pool=pool)
|
||||
return len(backends) > 0
|
||||
|
||||
def is_any_pool_available(self, canonical_model: str) -> bool:
|
||||
"""Check if any pool has healthy backends for a model."""
|
||||
for pool in self._pool_order:
|
||||
if self.is_pool_available(canonical_model, pool):
|
||||
return True
|
||||
return False
|
||||
Reference in New Issue
Block a user