""" MonitorWorker — periodically pings all monitor-enabled devices and manages alerts. Flow (every MONITOR_INTERVAL seconds): 1. Load all monitor_enabled=True devices from DB 2. Ping each device concurrently (semaphore-limited) 3. For each result: - Update device.status / last_seen / last_ping_ms - If offline and no open alert → create alert + SSE push - If online and open alerts exist → auto-resolve + SSE push 4. Publish SSE device_status event (frontend updates badge in real-time) SSE channel: task_id="__monitor__" (shared channel for all monitor events) """ import asyncio import time from datetime import datetime, timezone from sqlalchemy import select, update from app.models.alert import Alert from app.models.device import Device from app.services.database import AsyncSessionLocal from app.services.sse import SSEEvent, sse_manager # ── Configuration ────────────────────────────────────────────────────────────── MONITOR_INTERVAL = 60 # seconds between full check cycles PING_TIMEOUT = 3 # seconds per ping attempt PING_CONCURRENCY = 30 # max simultaneous pings MONITOR_CHANNEL = "__monitor__" # ── Lifecycle ────────────────────────────────────────────────────────────────── _stop_event: asyncio.Event | None = None _worker_task: asyncio.Task | None = None def start_monitor() -> None: global _stop_event, _worker_task _stop_event = asyncio.Event() _worker_task = asyncio.create_task(_monitor_loop()) print("[monitor] Worker started") async def stop_monitor() -> None: global _stop_event, _worker_task if _stop_event: _stop_event.set() if _worker_task: _worker_task.cancel() try: await _worker_task except asyncio.CancelledError: pass print("[monitor] Worker stopped") # ── Main loop ────────────────────────────────────────────────────────────────── async def _monitor_loop() -> None: assert _stop_event is not None while not _stop_event.is_set(): try: await _run_cycle() except Exception as exc: print(f"[monitor] Cycle error: {exc}") # Wait MONITOR_INTERVAL or until stop is requested try: await asyncio.wait_for(_stop_event.wait(), timeout=MONITOR_INTERVAL) except asyncio.TimeoutError: pass # normal — interval elapsed async def _run_cycle() -> None: """Load monitor-enabled devices and check each one.""" async with AsyncSessionLocal() as db: result = await db.execute( select(Device.id, Device.ip, Device.object_id) .where(Device.monitor_enabled == True, Device.is_active == True) ) rows = result.all() if not rows: return sem = asyncio.Semaphore(PING_CONCURRENCY) await asyncio.gather( *[_check_device(row.id, row.ip, row.object_id, sem) for row in rows], return_exceptions=True, ) # ── Per-device check ─────────────────────────────────────────────────────────── async def _check_device( device_id: int, ip: str, object_id: int, sem: asyncio.Semaphore, ) -> None: async with sem: success, ping_ms = await _ping(ip) now = datetime.now(timezone.utc) new_status = "online" if success else "offline" new_alert_id: int | None = None resolved_count = 0 async with AsyncSessionLocal() as db: async with db.begin(): device = await db.get(Device, device_id) if device is None: return device.status = new_status if success: device.last_seen = now device.last_ping_ms = ping_ms if not success: # Create alert only if none is already open/acknowledged existing = await db.execute( select(Alert) .where( Alert.device_id == device_id, Alert.status.in_(["open", "acknowledged"]), ) .limit(1) ) if existing.scalar_one_or_none() is None: alert = Alert( device_id=device_id, status="open", message=f"Устройство {ip} недоступно (ping failed)", created_at=now, ) db.add(alert) await db.flush() new_alert_id = alert.id else: # Auto-resolve any open / acknowledged alerts open_alerts_result = await db.execute( select(Alert).where( Alert.device_id == device_id, Alert.status.in_(["open", "acknowledged"]), ) ) open_alerts = open_alerts_result.scalars().all() for a in open_alerts: a.status = "resolved" a.resolved_at = now resolved_count += 1 # ── SSE publish (after DB commit) ───────────────────────────────────────── await sse_manager.publish( SSEEvent( type="device_status", task_id=MONITOR_CHANNEL, data={ "device_id": device_id, "object_id": object_id, "ip": ip, "status": new_status, "ping_ms": ping_ms if success else None, "ts": now.isoformat(), }, ) ) if new_alert_id is not None: await sse_manager.publish( SSEEvent( type="alert_created", task_id=MONITOR_CHANNEL, data={ "alert_id": new_alert_id, "device_id": device_id, "object_id": object_id, "ip": ip, }, ) ) if resolved_count > 0: await sse_manager.publish( SSEEvent( type="alert_updated", task_id=MONITOR_CHANNEL, data={ "device_id": device_id, "object_id": object_id, "resolved_count": resolved_count, }, ) ) # ── ICMP ping helper ─────────────────────────────────────────────────────────── async def _ping(ip: str) -> tuple[bool, int]: """ Returns (success, latency_ms). Uses system `ping` via subprocess — works in Linux containers. """ start = time.monotonic() try: proc = await asyncio.create_subprocess_exec( "ping", "-c", "1", "-W", str(PING_TIMEOUT), ip, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, ) await asyncio.wait_for(proc.wait(), timeout=PING_TIMEOUT + 2) ms = int((time.monotonic() - start) * 1000) return proc.returncode == 0, ms except asyncio.TimeoutError: return False, 0 except Exception: return False, 0