vincent
8531a3b595
- dashboard.html: major UX overhaul (+657/-308 lines) - server.py: /api/admin/backends now returns real-time RPM and model_count - pool_manager.py: random.shuffle backends for load distribution - config.py: health probe endpoint /v1/models → /models - docker-compose.yml: add SIDECAR_PRIMARY_WAIT_MAX_RETRIES=6 BIZ-52 post-review optimizations
86 lines
3.0 KiB
Python
86 lines
3.0 KiB
Python
"""Provider pool management: primary / fallback pool routing."""
|
|
|
|
import random
|
|
import structlog
|
|
from typing import Optional
|
|
|
|
from storage.backend_store import list_backends, get_pool_stats
|
|
from storage.models import Backend
|
|
|
|
logger = structlog.get_logger("sidecar_v2.pool_manager")
|
|
|
|
|
|
class PoolManager:
|
|
"""Manages provider pools and selects healthy backends for a given model.
|
|
|
|
Priority: primary pool → fallback pool.
|
|
Within a pool: healthy backends only, sorted by availability.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._pool_order = ["primary", "fallback"]
|
|
|
|
def get_available_backends(
|
|
self, canonical_model: str, pool: Optional[str] = None
|
|
) -> list[Backend]:
|
|
"""Get all healthy, enabled backends that serve a model, in pool order.
|
|
|
|
Args:
|
|
canonical_model: Canonical model name to match.
|
|
pool: Optional pool filter (primary/fallback). None = all pools.
|
|
|
|
Returns:
|
|
List of ready backends sorted by pool priority, then RPM utilization.
|
|
"""
|
|
backends: list[Backend] = []
|
|
|
|
pools_to_check = [pool] if pool else self._pool_order
|
|
for p in pools_to_check:
|
|
pool_backends = list_backends(pool=p, enabled_only=True, decrypt_key=True)
|
|
for b in pool_backends:
|
|
if b.status == "healthy" and b.has_model(canonical_model):
|
|
backends.append(b)
|
|
if pool:
|
|
break
|
|
|
|
random.shuffle(backends)
|
|
return backends
|
|
|
|
def get_any_healthy_backends(self, pool: Optional[str] = None) -> list[Backend]:
|
|
"""Get all healthy, enabled backends regardless of model."""
|
|
backends: list[Backend] = []
|
|
pools_to_check = [pool] if pool else self._pool_order
|
|
for p in pools_to_check:
|
|
pool_backends = list_backends(pool=p, enabled_only=True, decrypt_key=True)
|
|
for b in pool_backends:
|
|
if b.status == "healthy":
|
|
backends.append(b)
|
|
if pool:
|
|
break
|
|
random.shuffle(backends)
|
|
return backends
|
|
|
|
def get_pool_status(self) -> dict:
|
|
"""Get pool summary for dashboard."""
|
|
stats = get_pool_stats()
|
|
result = {}
|
|
for pool in self._pool_order:
|
|
s = stats.get(pool, {"total": 0, "enabled": 0, "healthy": 0, "cooling": 0, "error": 0})
|
|
result[pool] = s
|
|
# Also include any other pools
|
|
for pool, s in stats.items():
|
|
if pool not in result:
|
|
result[pool] = s
|
|
return result
|
|
|
|
def is_pool_available(self, canonical_model: str, pool: str = "primary") -> bool:
|
|
"""Check if a pool has any healthy backends for a model."""
|
|
backends = self.get_available_backends(canonical_model, pool=pool)
|
|
return len(backends) > 0
|
|
|
|
def is_any_pool_available(self, canonical_model: str) -> bool:
|
|
"""Check if any pool has healthy backends for a model."""
|
|
for pool in self._pool_order:
|
|
if self.is_pool_available(canonical_model, pool):
|
|
return True
|
|
return False |