127 lines
5.4 KiB
Python
127 lines
5.4 KiB
Python
import logging
|
|
import threading
|
|
import time
|
|
from typing import Dict, Any
|
|
import requests
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
|
|
@dataclass
|
|
class HealthMetrics:
|
|
last_check: float = 0
|
|
uptime_start: float = 0
|
|
gradio_endpoint_up: bool = False
|
|
api_endpoint_up: bool = False
|
|
consecutive_failures: int = 0
|
|
total_restarts: int = 0
|
|
response_times: Dict[str, float] = field(default_factory=dict)
|
|
|
|
class HealthMonitor:
|
|
def __init__(self, proxy_manager):
|
|
self.proxy_manager = proxy_manager
|
|
self.logger = logging.getLogger("ProxyHealthMonitor")
|
|
self.metrics = HealthMetrics()
|
|
self.monitoring = False
|
|
self.monitor_thread = None
|
|
self.check_interval = 30 # seconds
|
|
|
|
def start_monitoring(self):
|
|
"""Start health monitoring thread"""
|
|
if self.monitoring:
|
|
return
|
|
|
|
self.monitoring = True
|
|
self.metrics.uptime_start = time.time()
|
|
self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) # Make thread daemon
|
|
self.monitor_thread.start()
|
|
self.logger.info("Health monitoring started")
|
|
|
|
def stop_monitoring(self):
|
|
"""Stop health monitoring thread"""
|
|
if not self.monitoring:
|
|
return
|
|
|
|
self.monitoring = False
|
|
if self.monitor_thread and self.monitor_thread.is_alive():
|
|
try:
|
|
# Give the thread a chance to finish naturally
|
|
self.monitor_thread.join(timeout=5)
|
|
except Exception as e:
|
|
self.logger.warning(f"Could not gracefully stop monitoring thread: {e}")
|
|
finally:
|
|
self.monitor_thread = None
|
|
|
|
self.logger.info("Health monitoring stopped")
|
|
|
|
def _monitor_loop(self):
|
|
"""Main monitoring loop"""
|
|
while self.monitoring:
|
|
self._check_health()
|
|
time.sleep(self.check_interval)
|
|
|
|
def _check_health(self):
|
|
"""Check health of proxy endpoints"""
|
|
config = self.proxy_manager.config_manager.get_instance()
|
|
proxy_settings = config.proxy_settings
|
|
self.metrics.last_check = time.time()
|
|
|
|
try:
|
|
# Only check endpoints that are enabled and running
|
|
if proxy_settings.gradio_endpoint.enabled == "Enabled":
|
|
try:
|
|
start_time = time.time()
|
|
response = requests.get(
|
|
f"http://127.0.0.1:{config.gradio_port_number}/", # Use internal port
|
|
timeout=5
|
|
)
|
|
self.metrics.response_times['gradio'] = time.time() - start_time
|
|
self.metrics.gradio_endpoint_up = response.status_code == 200
|
|
except Exception as e:
|
|
self.logger.debug(f"Gradio endpoint check failed: {e}")
|
|
self.metrics.gradio_endpoint_up = False
|
|
|
|
if proxy_settings.api_endpoint.enabled == "Enabled":
|
|
try:
|
|
start_time = time.time()
|
|
response = requests.get(
|
|
f"http://127.0.0.1:{config.api_def.api_port_number}/", # Use internal port
|
|
timeout=5
|
|
)
|
|
self.metrics.response_times['api'] = time.time() - start_time
|
|
self.metrics.api_endpoint_up = response.status_code == 200
|
|
except Exception as e:
|
|
self.logger.debug(f"API endpoint check failed: {e}")
|
|
self.metrics.api_endpoint_up = False
|
|
|
|
if not self.metrics.gradio_endpoint_up and not self.metrics.api_endpoint_up:
|
|
self.metrics.consecutive_failures += 1
|
|
else:
|
|
self.metrics.consecutive_failures = 0
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Health check failed: {str(e)}")
|
|
self.metrics.consecutive_failures += 1
|
|
|
|
# Only attempt restart if we have multiple complete failures
|
|
if self.metrics.consecutive_failures >= 3:
|
|
self.logger.warning("Too many consecutive failures, attempting restart")
|
|
try:
|
|
if self.proxy_manager.stop_proxy() and self.proxy_manager.start_proxy():
|
|
self.metrics.total_restarts += 1
|
|
self.metrics.consecutive_failures = 0
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to restart proxy: {e}")
|
|
|
|
def get_health_status(self) -> Dict[str, Any]:
|
|
"""Get current health metrics"""
|
|
uptime = time.time() - self.metrics.uptime_start if self.metrics.uptime_start else 0
|
|
return {
|
|
"status": "healthy" if not self.metrics.consecutive_failures else "degraded",
|
|
"uptime": f"{int(uptime/3600)}h {int((uptime%3600)/60)}m",
|
|
"last_check": datetime.fromtimestamp(self.metrics.last_check).strftime('%Y-%m-%d %H:%M:%S')
|
|
if self.metrics.last_check else "Never",
|
|
"gradio_endpoint": "up" if self.metrics.gradio_endpoint_up else "down",
|
|
"api_endpoint": "up" if self.metrics.api_endpoint_up else "down",
|
|
"response_times": self.metrics.response_times,
|
|
"total_restarts": self.metrics.total_restarts
|
|
} |