utp_service/backend/app/workers/monitor.py

"""
MonitorWorker — periodically pings all monitor-enabled devices and manages alerts.

Flow (every MONITOR_INTERVAL seconds):
  1. Load all monitor_enabled=True devices from DB
  2. Ping each device concurrently (semaphore-limited)
  3. For each result:
     - Update device.status / last_seen / last_ping_ms
     - If offline and no open alert → create alert + SSE push
     - If online and open alerts exist → auto-resolve + SSE push
  4. Publish SSE device_status event (frontend updates badge in real-time)

SSE channel: task_id="__monitor__" (shared channel for all monitor events)
"""

import asyncio
import time
from datetime import datetime, timezone

from sqlalchemy import select, update

from app.models.alert import Alert
from app.models.device import Device
from app.services.database import AsyncSessionLocal
from app.services.sse import SSEEvent, sse_manager

# ── Configuration ──────────────────────────────────────────────────────────────
MONITOR_INTERVAL = 60     # seconds between full check cycles
PING_TIMEOUT = 3          # seconds per ping attempt
PING_CONCURRENCY = 30     # max simultaneous pings

MONITOR_CHANNEL = "__monitor__"

# ── Lifecycle ──────────────────────────────────────────────────────────────────
_stop_event: asyncio.Event | None = None
_worker_task: asyncio.Task | None = None


def start_monitor() -> None:
    global _stop_event, _worker_task
    _stop_event = asyncio.Event()
    _worker_task = asyncio.create_task(_monitor_loop())
    print("[monitor] Worker started")


async def stop_monitor() -> None:
    global _stop_event, _worker_task
    if _stop_event:
        _stop_event.set()
    if _worker_task:
        _worker_task.cancel()
        try:
            await _worker_task
        except asyncio.CancelledError:
            pass
    print("[monitor] Worker stopped")


# ── Main loop ──────────────────────────────────────────────────────────────────
async def _monitor_loop() -> None:
    assert _stop_event is not None
    while not _stop_event.is_set():
        try:
            await _run_cycle()
        except Exception as exc:
            print(f"[monitor] Cycle error: {exc}")
        # Wait MONITOR_INTERVAL or until stop is requested
        try:
            await asyncio.wait_for(_stop_event.wait(), timeout=MONITOR_INTERVAL)
        except asyncio.TimeoutError:
            pass  # normal — interval elapsed


async def _run_cycle() -> None:
    """Load monitor-enabled devices and check each one."""
    async with AsyncSessionLocal() as db:
        result = await db.execute(
            select(Device.id, Device.ip, Device.object_id)
            .where(Device.monitor_enabled == True, Device.is_active == True)
        )
        rows = result.all()

    if not rows:
        return

    sem = asyncio.Semaphore(PING_CONCURRENCY)
    await asyncio.gather(
        *[_check_device(row.id, row.ip, row.object_id, sem) for row in rows],
        return_exceptions=True,
    )


# ── Per-device check ───────────────────────────────────────────────────────────
async def _check_device(
    device_id: int,
    ip: str,
    object_id: int,
    sem: asyncio.Semaphore,
) -> None:
    async with sem:
        success, ping_ms = await _ping(ip)

    now = datetime.now(timezone.utc)
    new_status = "online" if success else "offline"
    new_alert_id: int | None = None
    resolved_count = 0

    async with AsyncSessionLocal() as db:
        async with db.begin():
            device = await db.get(Device, device_id)
            if device is None:
                return

            device.status = new_status
            device.last_ping_at = now
            if success:
                device.last_seen = now
                device.last_ping_ms = ping_ms

            if not success:
                # Create alert only if none is already open/acknowledged
                existing = await db.execute(
                    select(Alert)
                    .where(
                        Alert.device_id == device_id,
                        Alert.status.in_(["open", "acknowledged"]),
                    )
                    .limit(1)
                )
                if existing.scalar_one_or_none() is None:
                    alert = Alert(
                        device_id=device_id,
                        status="open",
                        message=f"Устройство {ip} недоступно (ping failed)",
                        created_at=now,
                    )
                    db.add(alert)
                    await db.flush()
                    new_alert_id = alert.id
            else:
                # Auto-resolve any open / acknowledged alerts
                open_alerts_result = await db.execute(
                    select(Alert).where(
                        Alert.device_id == device_id,
                        Alert.status.in_(["open", "acknowledged"]),
                    )
                )
                open_alerts = open_alerts_result.scalars().all()
                for a in open_alerts:
                    a.status = "resolved"
                    a.resolved_at = now
                    resolved_count += 1

    # ── SSE publish (after DB commit) ─────────────────────────────────────────
    await sse_manager.publish(
        SSEEvent(
            type="device_status",
            task_id=MONITOR_CHANNEL,
            data={
                "device_id": device_id,
                "object_id": object_id,
                "ip": ip,
                "status": new_status,
                "ping_ms": ping_ms if success else None,
                "ts": now.isoformat(),
            },
        )
    )

    if new_alert_id is not None:
        await sse_manager.publish(
            SSEEvent(
                type="alert_created",
                task_id=MONITOR_CHANNEL,
                data={
                    "alert_id": new_alert_id,
                    "device_id": device_id,
                    "object_id": object_id,
                    "ip": ip,
                },
            )
        )

    if resolved_count > 0:
        await sse_manager.publish(
            SSEEvent(
                type="alert_updated",
                task_id=MONITOR_CHANNEL,
                data={
                    "device_id": device_id,
                    "object_id": object_id,
                    "resolved_count": resolved_count,
                },
            )
        )


# ── ICMP ping helper ───────────────────────────────────────────────────────────
async def _ping(ip: str) -> tuple[bool, int]:
    """
    Returns (success, latency_ms).
    Uses system `ping` via subprocess — works in Linux containers.
    """
    start = time.monotonic()
    try:
        proc = await asyncio.create_subprocess_exec(
            "ping", "-c", "1", "-W", str(PING_TIMEOUT), ip,
            stdout=asyncio.subprocess.DEVNULL,
            stderr=asyncio.subprocess.DEVNULL,
        )
        await asyncio.wait_for(proc.wait(), timeout=PING_TIMEOUT + 2)
        ms = int((time.monotonic() - start) * 1000)
        return proc.returncode == 0, ms
    except asyncio.TimeoutError:
        return False, 0
    except Exception:
        return False, 0