vincent
2d95ae50a5
- proxy.py: Fix route path duplication (v1/v1 → v1) when upstream base URL already includes /v1 prefix - proxy.py: Fix _emergency_count global variable for metrics tracking - server.py: Add logging.basicConfig(level=logging.INFO) for structlog INFO-level log visibility - Full multi-pool routing: primary → fallback → emergency passthrough - Per-backend rate limiting with RPM-based token bucket - 429 cooldown mechanism with automatic recovery - Dashboard with SSE real-time monitoring - Admin API for backend/pool/config management - SQLite-backed persistence with encrypted API key storage - Docker compose deployment Deployed by opengineer 严维序 as BIZ-50 Step 4
83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
"""Provider pool management: primary / fallback pool routing."""
|
|
|
|
import structlog
|
|
from typing import Optional
|
|
|
|
from storage.backend_store import list_backends, get_pool_stats
|
|
from storage.models import Backend
|
|
|
|
logger = structlog.get_logger("sidecar_v2.pool_manager")
|
|
|
|
|
|
class PoolManager:
|
|
"""Manages provider pools and selects healthy backends for a given model.
|
|
|
|
Priority: primary pool → fallback pool.
|
|
Within a pool: healthy backends only, sorted by availability.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._pool_order = ["primary", "fallback"]
|
|
|
|
def get_available_backends(
|
|
self, canonical_model: str, pool: Optional[str] = None
|
|
) -> list[Backend]:
|
|
"""Get all healthy, enabled backends that serve a model, in pool order.
|
|
|
|
Args:
|
|
canonical_model: Canonical model name to match.
|
|
pool: Optional pool filter (primary/fallback). None = all pools.
|
|
|
|
Returns:
|
|
List of ready backends sorted by pool priority, then RPM utilization.
|
|
"""
|
|
backends: list[Backend] = []
|
|
|
|
pools_to_check = [pool] if pool else self._pool_order
|
|
for p in pools_to_check:
|
|
pool_backends = list_backends(pool=p, enabled_only=True, decrypt_key=True)
|
|
for b in pool_backends:
|
|
if b.status == "healthy" and b.has_model(canonical_model):
|
|
backends.append(b)
|
|
if pool:
|
|
break
|
|
|
|
return backends
|
|
|
|
def get_any_healthy_backends(self, pool: Optional[str] = None) -> list[Backend]:
|
|
"""Get all healthy, enabled backends regardless of model."""
|
|
backends: list[Backend] = []
|
|
pools_to_check = [pool] if pool else self._pool_order
|
|
for p in pools_to_check:
|
|
pool_backends = list_backends(pool=p, enabled_only=True, decrypt_key=True)
|
|
for b in pool_backends:
|
|
if b.status == "healthy":
|
|
backends.append(b)
|
|
if pool:
|
|
break
|
|
return backends
|
|
|
|
def get_pool_status(self) -> dict:
|
|
"""Get pool summary for dashboard."""
|
|
stats = get_pool_stats()
|
|
result = {}
|
|
for pool in self._pool_order:
|
|
s = stats.get(pool, {"total": 0, "enabled": 0, "healthy": 0, "cooling": 0, "error": 0})
|
|
result[pool] = s
|
|
# Also include any other pools
|
|
for pool, s in stats.items():
|
|
if pool not in result:
|
|
result[pool] = s
|
|
return result
|
|
|
|
def is_pool_available(self, canonical_model: str, pool: str = "primary") -> bool:
|
|
"""Check if a pool has any healthy backends for a model."""
|
|
backends = self.get_available_backends(canonical_model, pool=pool)
|
|
return len(backends) > 0
|
|
|
|
def is_any_pool_available(self, canonical_model: str) -> bool:
|
|
"""Check if any pool has healthy backends for a model."""
|
|
for pool in self._pool_order:
|
|
if self.is_pool_available(canonical_model, pool):
|
|
return True
|
|
return False |