этап4
This commit is contained in:
parent
87bf93de89
commit
6b36485718
12 changed files with 899 additions and 4 deletions
47
backend/alembic/versions/0006_alerts.py
Normal file
47
backend/alembic/versions/0006_alerts.py
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
"""0006 alerts
|
||||||
|
|
||||||
|
Revision ID: 0006
|
||||||
|
Revises: 0005
|
||||||
|
Create Date: 2026-04-07
|
||||||
|
"""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = "0006"
|
||||||
|
down_revision = "0005"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"alerts",
|
||||||
|
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"device_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("devices.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("status", sa.String(20), nullable=False, server_default="open"),
|
||||||
|
sa.Column("message", sa.Text(), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("acknowledged_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"acknowledged_by",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_alerts_device_status", "alerts", ["device_id", "status"])
|
||||||
|
op.create_index("ix_alerts_status_created", "alerts", ["status", "created_at"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_alerts_status_created", table_name="alerts")
|
||||||
|
op.drop_index("ix_alerts_device_status", table_name="alerts")
|
||||||
|
op.drop_table("alerts")
|
||||||
|
|
@ -6,7 +6,8 @@ from fastapi.openapi.utils import get_openapi
|
||||||
|
|
||||||
from app.middleware.auth import AuthMiddleware
|
from app.middleware.auth import AuthMiddleware
|
||||||
from app.plugins.loader import load_plugins
|
from app.plugins.loader import load_plugins
|
||||||
from app.routers import auth, devices, events, objects, racks, snapshots, tags, tasks, zones
|
from app.routers import alerts, auth, devices, events, objects, racks, snapshots, tags, tasks, zones
|
||||||
|
from app.workers.monitor import start_monitor, stop_monitor
|
||||||
from app.workers.startup import recover_stale_tasks
|
from app.workers.startup import recover_stale_tasks
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -14,7 +15,9 @@ from app.workers.startup import recover_stale_tasks
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
load_plugins()
|
load_plugins()
|
||||||
await recover_stale_tasks()
|
await recover_stale_tasks()
|
||||||
|
start_monitor()
|
||||||
yield
|
yield
|
||||||
|
await stop_monitor()
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
|
|
@ -42,6 +45,7 @@ def create_app() -> FastAPI:
|
||||||
app.include_router(racks.router)
|
app.include_router(racks.router)
|
||||||
app.include_router(zones.router)
|
app.include_router(zones.router)
|
||||||
app.include_router(tags.router)
|
app.include_router(tags.router)
|
||||||
|
app.include_router(alerts.router)
|
||||||
app.include_router(snapshots.router)
|
app.include_router(snapshots.router)
|
||||||
app.include_router(tasks.router)
|
app.include_router(tasks.router)
|
||||||
app.include_router(events.router)
|
app.include_router(events.router)
|
||||||
|
|
|
||||||
35
backend/app/models/alert.py
Normal file
35
backend/app/models/alert.py
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, ForeignKey, Integer, String, Text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class Alert(Base):
|
||||||
|
__tablename__ = "alerts"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer(), autoincrement=True, primary_key=True)
|
||||||
|
device_id: Mapped[int] = mapped_column(
|
||||||
|
Integer(), ForeignKey("devices.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
# "open" | "acknowledged" | "resolved"
|
||||||
|
status: Mapped[str] = mapped_column(String(20), nullable=False, default="open")
|
||||||
|
message: Mapped[str] = mapped_column(Text(), nullable=False)
|
||||||
|
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
acknowledged_at: Mapped[Optional[datetime]] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
acknowledged_by: Mapped[Optional[int]] = mapped_column(
|
||||||
|
Integer(), ForeignKey("users.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
resolved_at: Mapped[Optional[datetime]] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
device: Mapped["Device"] = relationship("Device")
|
||||||
|
acknowledger: Mapped[Optional["User"]] = relationship(
|
||||||
|
"User", foreign_keys=[acknowledged_by]
|
||||||
|
)
|
||||||
158
backend/app/routers/alerts.py
Normal file
158
backend/app/routers/alerts.py
Normal file
|
|
@ -0,0 +1,158 @@
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
|
from app.dependencies import get_current_user, get_db
|
||||||
|
from app.models.alert import Alert
|
||||||
|
from app.models.device import Device
|
||||||
|
from app.models.user import User
|
||||||
|
from app.schemas.alert import AlertRead
|
||||||
|
from app.services.sse import SSEEvent, sse_manager
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1/alerts", tags=["alerts"])
|
||||||
|
|
||||||
|
# SSE channel for all monitor/alert events
|
||||||
|
MONITOR_CHANNEL = "__monitor__"
|
||||||
|
|
||||||
|
|
||||||
|
def _to_read(alert: Alert) -> AlertRead:
|
||||||
|
return AlertRead(
|
||||||
|
id=alert.id,
|
||||||
|
device_id=alert.device_id,
|
||||||
|
device_ip=alert.device.ip if alert.device else None,
|
||||||
|
device_hostname=alert.device.hostname if alert.device else None,
|
||||||
|
object_id=alert.device.object_id if alert.device else None,
|
||||||
|
object_name=(
|
||||||
|
alert.device.object.name
|
||||||
|
if alert.device and alert.device.object
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
status=alert.status,
|
||||||
|
message=alert.message,
|
||||||
|
created_at=alert.created_at,
|
||||||
|
acknowledged_at=alert.acknowledged_at,
|
||||||
|
acknowledged_by_username=(
|
||||||
|
alert.acknowledger.username if alert.acknowledger else None
|
||||||
|
),
|
||||||
|
resolved_at=alert.resolved_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _load_alert(db: AsyncSession, alert_id: int) -> Alert:
|
||||||
|
result = await db.execute(
|
||||||
|
select(Alert)
|
||||||
|
.options(
|
||||||
|
selectinload(Alert.device).selectinload(Device.object),
|
||||||
|
selectinload(Alert.acknowledger),
|
||||||
|
)
|
||||||
|
.where(Alert.id == alert_id)
|
||||||
|
)
|
||||||
|
alert = result.scalar_one_or_none()
|
||||||
|
if alert is None:
|
||||||
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Alert not found")
|
||||||
|
return alert
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=list[AlertRead])
|
||||||
|
async def list_alerts(
|
||||||
|
alert_status: Optional[str] = Query(None, alias="status"),
|
||||||
|
object_id: Optional[int] = Query(None),
|
||||||
|
device_id: Optional[int] = Query(None),
|
||||||
|
limit: int = Query(100, ge=1, le=500),
|
||||||
|
skip: int = Query(0, ge=0),
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
_: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
query = (
|
||||||
|
select(Alert)
|
||||||
|
.options(
|
||||||
|
selectinload(Alert.device).selectinload(Device.object),
|
||||||
|
selectinload(Alert.acknowledger),
|
||||||
|
)
|
||||||
|
.order_by(Alert.created_at.desc())
|
||||||
|
.offset(skip)
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
|
||||||
|
if alert_status and alert_status != "all":
|
||||||
|
query = query.where(Alert.status == alert_status)
|
||||||
|
if device_id is not None:
|
||||||
|
query = query.where(Alert.device_id == device_id)
|
||||||
|
if object_id is not None:
|
||||||
|
query = query.join(Device, Alert.device_id == Device.id).where(
|
||||||
|
Device.object_id == object_id
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await db.execute(query)
|
||||||
|
return [_to_read(a) for a in result.scalars().all()]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{alert_id}/acknowledge", response_model=AlertRead)
|
||||||
|
async def acknowledge_alert(
|
||||||
|
alert_id: int,
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
alert = await _load_alert(db, alert_id)
|
||||||
|
if alert.status not in ("open",):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail=f"Cannot acknowledge alert with status '{alert.status}'",
|
||||||
|
)
|
||||||
|
alert.status = "acknowledged"
|
||||||
|
alert.acknowledged_at = datetime.now(timezone.utc)
|
||||||
|
alert.acknowledged_by = current_user.id
|
||||||
|
await db.flush()
|
||||||
|
await db.refresh(alert)
|
||||||
|
# Reload relationships after flush
|
||||||
|
alert = await _load_alert(db, alert_id)
|
||||||
|
result = _to_read(alert)
|
||||||
|
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="alert_updated",
|
||||||
|
task_id=MONITOR_CHANNEL,
|
||||||
|
data={
|
||||||
|
"alert_id": alert_id,
|
||||||
|
"status": "acknowledged",
|
||||||
|
"device_id": alert.device_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{alert_id}/resolve", response_model=AlertRead)
|
||||||
|
async def resolve_alert(
|
||||||
|
alert_id: int,
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
_: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
alert = await _load_alert(db, alert_id)
|
||||||
|
if alert.status == "resolved":
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail="Alert is already resolved",
|
||||||
|
)
|
||||||
|
alert.status = "resolved"
|
||||||
|
alert.resolved_at = datetime.now(timezone.utc)
|
||||||
|
await db.flush()
|
||||||
|
alert = await _load_alert(db, alert_id)
|
||||||
|
result = _to_read(alert)
|
||||||
|
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="alert_updated",
|
||||||
|
task_id=MONITOR_CHANNEL,
|
||||||
|
data={
|
||||||
|
"alert_id": alert_id,
|
||||||
|
"status": "resolved",
|
||||||
|
"device_id": alert.device_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return result
|
||||||
21
backend/app/schemas/alert.py
Normal file
21
backend/app/schemas/alert.py
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict
|
||||||
|
|
||||||
|
|
||||||
|
class AlertRead(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
id: int
|
||||||
|
device_id: int
|
||||||
|
device_ip: Optional[str] = None
|
||||||
|
device_hostname: Optional[str] = None
|
||||||
|
object_id: Optional[int] = None
|
||||||
|
object_name: Optional[str] = None
|
||||||
|
status: str
|
||||||
|
message: str
|
||||||
|
created_at: datetime
|
||||||
|
acknowledged_at: Optional[datetime] = None
|
||||||
|
acknowledged_by_username: Optional[str] = None
|
||||||
|
resolved_at: Optional[datetime] = None
|
||||||
216
backend/app/workers/monitor.py
Normal file
216
backend/app/workers/monitor.py
Normal file
|
|
@ -0,0 +1,216 @@
|
||||||
|
"""
|
||||||
|
MonitorWorker — periodically pings all monitor-enabled devices and manages alerts.
|
||||||
|
|
||||||
|
Flow (every MONITOR_INTERVAL seconds):
|
||||||
|
1. Load all monitor_enabled=True devices from DB
|
||||||
|
2. Ping each device concurrently (semaphore-limited)
|
||||||
|
3. For each result:
|
||||||
|
- Update device.status / last_seen / last_ping_ms
|
||||||
|
- If offline and no open alert → create alert + SSE push
|
||||||
|
- If online and open alerts exist → auto-resolve + SSE push
|
||||||
|
4. Publish SSE device_status event (frontend updates badge in real-time)
|
||||||
|
|
||||||
|
SSE channel: task_id="__monitor__" (shared channel for all monitor events)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
|
from app.models.alert import Alert
|
||||||
|
from app.models.device import Device
|
||||||
|
from app.services.database import AsyncSessionLocal
|
||||||
|
from app.services.sse import SSEEvent, sse_manager
|
||||||
|
|
||||||
|
# ── Configuration ──────────────────────────────────────────────────────────────
|
||||||
|
MONITOR_INTERVAL = 60 # seconds between full check cycles
|
||||||
|
PING_TIMEOUT = 3 # seconds per ping attempt
|
||||||
|
PING_CONCURRENCY = 30 # max simultaneous pings
|
||||||
|
|
||||||
|
MONITOR_CHANNEL = "__monitor__"
|
||||||
|
|
||||||
|
# ── Lifecycle ──────────────────────────────────────────────────────────────────
|
||||||
|
_stop_event: asyncio.Event | None = None
|
||||||
|
_worker_task: asyncio.Task | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def start_monitor() -> None:
|
||||||
|
global _stop_event, _worker_task
|
||||||
|
_stop_event = asyncio.Event()
|
||||||
|
_worker_task = asyncio.create_task(_monitor_loop())
|
||||||
|
print("[monitor] Worker started")
|
||||||
|
|
||||||
|
|
||||||
|
async def stop_monitor() -> None:
|
||||||
|
global _stop_event, _worker_task
|
||||||
|
if _stop_event:
|
||||||
|
_stop_event.set()
|
||||||
|
if _worker_task:
|
||||||
|
_worker_task.cancel()
|
||||||
|
try:
|
||||||
|
await _worker_task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
print("[monitor] Worker stopped")
|
||||||
|
|
||||||
|
|
||||||
|
# ── Main loop ──────────────────────────────────────────────────────────────────
|
||||||
|
async def _monitor_loop() -> None:
|
||||||
|
assert _stop_event is not None
|
||||||
|
while not _stop_event.is_set():
|
||||||
|
try:
|
||||||
|
await _run_cycle()
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"[monitor] Cycle error: {exc}")
|
||||||
|
# Wait MONITOR_INTERVAL or until stop is requested
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(_stop_event.wait(), timeout=MONITOR_INTERVAL)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass # normal — interval elapsed
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_cycle() -> None:
|
||||||
|
"""Load monitor-enabled devices and check each one."""
|
||||||
|
async with AsyncSessionLocal() as db:
|
||||||
|
result = await db.execute(
|
||||||
|
select(Device.id, Device.ip, Device.object_id)
|
||||||
|
.where(Device.monitor_enabled == True, Device.is_active == True)
|
||||||
|
)
|
||||||
|
rows = result.all()
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
|
||||||
|
sem = asyncio.Semaphore(PING_CONCURRENCY)
|
||||||
|
await asyncio.gather(
|
||||||
|
*[_check_device(row.id, row.ip, row.object_id, sem) for row in rows],
|
||||||
|
return_exceptions=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Per-device check ───────────────────────────────────────────────────────────
|
||||||
|
async def _check_device(
|
||||||
|
device_id: int,
|
||||||
|
ip: str,
|
||||||
|
object_id: int,
|
||||||
|
sem: asyncio.Semaphore,
|
||||||
|
) -> None:
|
||||||
|
async with sem:
|
||||||
|
success, ping_ms = await _ping(ip)
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
new_status = "online" if success else "offline"
|
||||||
|
new_alert_id: int | None = None
|
||||||
|
resolved_count = 0
|
||||||
|
|
||||||
|
async with AsyncSessionLocal() as db:
|
||||||
|
async with db.begin():
|
||||||
|
device = await db.get(Device, device_id)
|
||||||
|
if device is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
device.status = new_status
|
||||||
|
if success:
|
||||||
|
device.last_seen = now
|
||||||
|
device.last_ping_ms = ping_ms
|
||||||
|
|
||||||
|
if not success:
|
||||||
|
# Create alert only if none is already open/acknowledged
|
||||||
|
existing = await db.execute(
|
||||||
|
select(Alert)
|
||||||
|
.where(
|
||||||
|
Alert.device_id == device_id,
|
||||||
|
Alert.status.in_(["open", "acknowledged"]),
|
||||||
|
)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
if existing.scalar_one_or_none() is None:
|
||||||
|
alert = Alert(
|
||||||
|
device_id=device_id,
|
||||||
|
status="open",
|
||||||
|
message=f"Устройство {ip} недоступно (ping failed)",
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
db.add(alert)
|
||||||
|
await db.flush()
|
||||||
|
new_alert_id = alert.id
|
||||||
|
else:
|
||||||
|
# Auto-resolve any open / acknowledged alerts
|
||||||
|
open_alerts_result = await db.execute(
|
||||||
|
select(Alert).where(
|
||||||
|
Alert.device_id == device_id,
|
||||||
|
Alert.status.in_(["open", "acknowledged"]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
open_alerts = open_alerts_result.scalars().all()
|
||||||
|
for a in open_alerts:
|
||||||
|
a.status = "resolved"
|
||||||
|
a.resolved_at = now
|
||||||
|
resolved_count += 1
|
||||||
|
|
||||||
|
# ── SSE publish (after DB commit) ─────────────────────────────────────────
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="device_status",
|
||||||
|
task_id=MONITOR_CHANNEL,
|
||||||
|
data={
|
||||||
|
"device_id": device_id,
|
||||||
|
"object_id": object_id,
|
||||||
|
"ip": ip,
|
||||||
|
"status": new_status,
|
||||||
|
"ping_ms": ping_ms if success else None,
|
||||||
|
"ts": now.isoformat(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if new_alert_id is not None:
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="alert_created",
|
||||||
|
task_id=MONITOR_CHANNEL,
|
||||||
|
data={
|
||||||
|
"alert_id": new_alert_id,
|
||||||
|
"device_id": device_id,
|
||||||
|
"object_id": object_id,
|
||||||
|
"ip": ip,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if resolved_count > 0:
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="alert_updated",
|
||||||
|
task_id=MONITOR_CHANNEL,
|
||||||
|
data={
|
||||||
|
"device_id": device_id,
|
||||||
|
"object_id": object_id,
|
||||||
|
"resolved_count": resolved_count,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── ICMP ping helper ───────────────────────────────────────────────────────────
|
||||||
|
async def _ping(ip: str) -> tuple[bool, int]:
|
||||||
|
"""
|
||||||
|
Returns (success, latency_ms).
|
||||||
|
Uses system `ping` via subprocess — works in Linux containers.
|
||||||
|
"""
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
proc = await asyncio.create_subprocess_exec(
|
||||||
|
"ping", "-c", "1", "-W", str(PING_TIMEOUT), ip,
|
||||||
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
await asyncio.wait_for(proc.wait(), timeout=PING_TIMEOUT + 2)
|
||||||
|
ms = int((time.monotonic() - start) * 1000)
|
||||||
|
return proc.returncode == 0, ms
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
return False, 0
|
||||||
|
except Exception:
|
||||||
|
return False, 0
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import { Navigate, Route, Routes } from 'react-router-dom'
|
import { Navigate, Route, Routes } from 'react-router-dom'
|
||||||
import { AppLayout } from './components/AppLayout'
|
import { AppLayout } from './components/AppLayout'
|
||||||
|
import { AlertsPage } from './pages/Alerts'
|
||||||
import { LoginPage } from './pages/Login'
|
import { LoginPage } from './pages/Login'
|
||||||
import { ObjectDetailPage } from './pages/ObjectDetail'
|
import { ObjectDetailPage } from './pages/ObjectDetail'
|
||||||
import { ObjectsPage } from './pages/Objects'
|
import { ObjectsPage } from './pages/Objects'
|
||||||
|
|
@ -26,6 +27,7 @@ export default function App() {
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<Route index element={<Navigate to="/objects" replace />} />
|
<Route index element={<Navigate to="/objects" replace />} />
|
||||||
|
<Route path="alerts" element={<AlertsPage />} />
|
||||||
<Route path="objects" element={<ObjectsPage />} />
|
<Route path="objects" element={<ObjectsPage />} />
|
||||||
<Route path="objects/:id" element={<ObjectDetailPage />} />
|
<Route path="objects/:id" element={<ObjectDetailPage />} />
|
||||||
<Route path="objects/:id/snapshots" element={<SnapshotsPage />} />
|
<Route path="objects/:id/snapshots" element={<SnapshotsPage />} />
|
||||||
|
|
|
||||||
52
frontend/src/api/alerts.ts
Normal file
52
frontend/src/api/alerts.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||||
|
import { api } from './client'
|
||||||
|
|
||||||
|
export interface AlertItem {
|
||||||
|
id: number
|
||||||
|
device_id: number
|
||||||
|
device_ip: string | null
|
||||||
|
device_hostname: string | null
|
||||||
|
object_id: number | null
|
||||||
|
object_name: string | null
|
||||||
|
status: 'open' | 'acknowledged' | 'resolved'
|
||||||
|
message: string
|
||||||
|
created_at: string
|
||||||
|
acknowledged_at: string | null
|
||||||
|
acknowledged_by_username: string | null
|
||||||
|
resolved_at: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useAlerts = (params?: {
|
||||||
|
status?: string
|
||||||
|
object_id?: number
|
||||||
|
device_id?: number
|
||||||
|
limit?: number
|
||||||
|
skip?: number
|
||||||
|
refetchInterval?: number
|
||||||
|
}) => {
|
||||||
|
const { refetchInterval, ...queryParams } = params ?? {}
|
||||||
|
return useQuery({
|
||||||
|
queryKey: ['alerts', queryParams],
|
||||||
|
queryFn: () =>
|
||||||
|
api.get<AlertItem[]>('/api/v1/alerts', { params: queryParams }).then((r) => r.data),
|
||||||
|
refetchInterval: refetchInterval ?? false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useAcknowledgeAlert = () => {
|
||||||
|
const qc = useQueryClient()
|
||||||
|
return useMutation({
|
||||||
|
mutationFn: (id: number) =>
|
||||||
|
api.post<AlertItem>(`/api/v1/alerts/${id}/acknowledge`).then((r) => r.data),
|
||||||
|
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export const useResolveAlert = () => {
|
||||||
|
const qc = useQueryClient()
|
||||||
|
return useMutation({
|
||||||
|
mutationFn: (id: number) =>
|
||||||
|
api.post<AlertItem>(`/api/v1/alerts/${id}/resolve`).then((r) => r.data),
|
||||||
|
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -1,21 +1,48 @@
|
||||||
import {
|
import {
|
||||||
ApartmentOutlined,
|
ApartmentOutlined,
|
||||||
|
BellOutlined,
|
||||||
LogoutOutlined,
|
LogoutOutlined,
|
||||||
ThunderboltOutlined,
|
ThunderboltOutlined,
|
||||||
} from '@ant-design/icons'
|
} from '@ant-design/icons'
|
||||||
import { Layout, Menu, Typography } from 'antd'
|
import { Badge, Layout, Menu, Typography } from 'antd'
|
||||||
import { useNavigate, useLocation, Outlet } from 'react-router-dom'
|
import { useNavigate, useLocation, Outlet } from 'react-router-dom'
|
||||||
|
import { useQueryClient } from '@tanstack/react-query'
|
||||||
import { useAuthStore } from '../store/auth'
|
import { useAuthStore } from '../store/auth'
|
||||||
import { api } from '../api/client'
|
import { api } from '../api/client'
|
||||||
|
import { useAlerts } from '../api/alerts'
|
||||||
|
import { useMonitorSSE } from '../hooks/useMonitorSSE'
|
||||||
|
|
||||||
const { Sider, Content, Header } = Layout
|
const { Sider, Content } = Layout
|
||||||
|
|
||||||
export function AppLayout() {
|
export function AppLayout() {
|
||||||
const navigate = useNavigate()
|
const navigate = useNavigate()
|
||||||
const location = useLocation()
|
const location = useLocation()
|
||||||
|
const qc = useQueryClient()
|
||||||
const { username, refreshToken, clearTokens } = useAuthStore()
|
const { username, refreshToken, clearTokens } = useAuthStore()
|
||||||
|
|
||||||
const selectedKey = location.pathname.startsWith('/tasks') ? 'tasks' : 'objects'
|
// Open alert count for sidebar badge — refresh every 60s
|
||||||
|
const { data: openAlerts } = useAlerts({
|
||||||
|
status: 'open',
|
||||||
|
limit: 500,
|
||||||
|
refetchInterval: 60_000,
|
||||||
|
})
|
||||||
|
const openCount = openAlerts?.length ?? 0
|
||||||
|
|
||||||
|
// Global monitor SSE — invalidates caches when monitor events arrive
|
||||||
|
useMonitorSSE({
|
||||||
|
onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
|
onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
|
onDeviceStatus: (ev) => {
|
||||||
|
// Invalidate devices for the affected object so status badge updates live
|
||||||
|
qc.invalidateQueries({ queryKey: ['devices', ev.object_id] })
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const selectedKey = location.pathname.startsWith('/tasks')
|
||||||
|
? 'tasks'
|
||||||
|
: location.pathname.startsWith('/alerts')
|
||||||
|
? 'alerts'
|
||||||
|
: 'objects'
|
||||||
|
|
||||||
const handleLogout = async () => {
|
const handleLogout = async () => {
|
||||||
try {
|
try {
|
||||||
|
|
@ -45,6 +72,19 @@ export function AppLayout() {
|
||||||
label: 'Объекты',
|
label: 'Объекты',
|
||||||
onClick: () => navigate('/objects'),
|
onClick: () => navigate('/objects'),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: 'alerts',
|
||||||
|
icon: <BellOutlined />,
|
||||||
|
label: (
|
||||||
|
<span>
|
||||||
|
Алерты
|
||||||
|
{openCount > 0 && (
|
||||||
|
<Badge count={openCount} size="small" style={{ marginLeft: 8 }} />
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
),
|
||||||
|
onClick: () => navigate('/alerts'),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
key: 'tasks',
|
key: 'tasks',
|
||||||
icon: <ThunderboltOutlined />,
|
icon: <ThunderboltOutlined />,
|
||||||
|
|
|
||||||
75
frontend/src/hooks/useMonitorSSE.ts
Normal file
75
frontend/src/hooks/useMonitorSSE.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
/**
|
||||||
|
* useMonitorSSE — subscribes to the global monitor SSE channel ("__monitor__").
|
||||||
|
*
|
||||||
|
* The backend publishes three event types on this channel:
|
||||||
|
* device_status — { device_id, object_id, ip, status, ping_ms, ts }
|
||||||
|
* alert_created — { alert_id, device_id, object_id, ip }
|
||||||
|
* alert_updated — { alert_id?, device_id, resolved_count? }
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { fetchEventSource } from '@microsoft/fetch-event-source'
|
||||||
|
import { useEffect, useRef } from 'react'
|
||||||
|
import { useAuthStore } from '../store/auth'
|
||||||
|
|
||||||
|
export interface DeviceStatusEvent {
|
||||||
|
device_id: number
|
||||||
|
object_id: number
|
||||||
|
ip: string
|
||||||
|
status: 'online' | 'offline'
|
||||||
|
ping_ms: number | null
|
||||||
|
ts: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AlertCreatedEvent {
|
||||||
|
alert_id: number
|
||||||
|
device_id: number
|
||||||
|
object_id: number
|
||||||
|
ip: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AlertUpdatedEvent {
|
||||||
|
alert_id?: number
|
||||||
|
device_id: number
|
||||||
|
status?: string
|
||||||
|
resolved_count?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MonitorHandlers {
|
||||||
|
onDeviceStatus?: (data: DeviceStatusEvent) => void
|
||||||
|
onAlertCreated?: (data: AlertCreatedEvent) => void
|
||||||
|
onAlertUpdated?: (data: AlertUpdatedEvent) => void
|
||||||
|
}
|
||||||
|
|
||||||
|
const MONITOR_CHANNEL = '__monitor__'
|
||||||
|
|
||||||
|
export function useMonitorSSE(handlers: MonitorHandlers) {
|
||||||
|
const accessToken = useAuthStore((s) => s.accessToken)
|
||||||
|
const handlersRef = useRef(handlers)
|
||||||
|
handlersRef.current = handlers
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!accessToken) return
|
||||||
|
const ctrl = new AbortController()
|
||||||
|
|
||||||
|
fetchEventSource(`/api/v1/events?task_id=${MONITOR_CHANNEL}`, {
|
||||||
|
headers: { Authorization: `Bearer ${accessToken}` },
|
||||||
|
signal: ctrl.signal,
|
||||||
|
onmessage(ev) {
|
||||||
|
try {
|
||||||
|
const data = JSON.parse(ev.data)
|
||||||
|
if (ev.event === 'device_status') handlersRef.current.onDeviceStatus?.(data)
|
||||||
|
else if (ev.event === 'alert_created') handlersRef.current.onAlertCreated?.(data)
|
||||||
|
else if (ev.event === 'alert_updated') handlersRef.current.onAlertUpdated?.(data)
|
||||||
|
} catch {
|
||||||
|
// ignore parse errors
|
||||||
|
}
|
||||||
|
},
|
||||||
|
onerror() {
|
||||||
|
// fetchEventSource retries automatically; abort only on explicit error
|
||||||
|
ctrl.abort()
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return () => ctrl.abort()
|
||||||
|
}, [accessToken])
|
||||||
|
}
|
||||||
239
frontend/src/pages/Alerts.tsx
Normal file
239
frontend/src/pages/Alerts.tsx
Normal file
|
|
@ -0,0 +1,239 @@
|
||||||
|
import {
|
||||||
|
CheckCircleOutlined,
|
||||||
|
ClockCircleOutlined,
|
||||||
|
ExclamationCircleOutlined,
|
||||||
|
} from '@ant-design/icons'
|
||||||
|
import {
|
||||||
|
Badge,
|
||||||
|
Breadcrumb,
|
||||||
|
Button,
|
||||||
|
Col,
|
||||||
|
Popconfirm,
|
||||||
|
Row,
|
||||||
|
Select,
|
||||||
|
Space,
|
||||||
|
Table,
|
||||||
|
Tag,
|
||||||
|
Tooltip,
|
||||||
|
Typography,
|
||||||
|
message,
|
||||||
|
} from 'antd'
|
||||||
|
import { useState } from 'react'
|
||||||
|
import { useQueryClient } from '@tanstack/react-query'
|
||||||
|
import { useNavigate } from 'react-router-dom'
|
||||||
|
import {
|
||||||
|
useAlerts,
|
||||||
|
useAcknowledgeAlert,
|
||||||
|
useResolveAlert,
|
||||||
|
type AlertItem,
|
||||||
|
} from '../api/alerts'
|
||||||
|
import { useMonitorSSE } from '../hooks/useMonitorSSE'
|
||||||
|
import dayjs from 'dayjs'
|
||||||
|
|
||||||
|
const STATUS_CONFIG: Record<
|
||||||
|
string,
|
||||||
|
{ label: string; color: string; icon: React.ReactNode }
|
||||||
|
> = {
|
||||||
|
open: { label: 'Открыт', color: 'error', icon: <ExclamationCircleOutlined /> },
|
||||||
|
acknowledged: { label: 'Принят', color: 'warning', icon: <ClockCircleOutlined /> },
|
||||||
|
resolved: { label: 'Решён', color: 'success', icon: <CheckCircleOutlined /> },
|
||||||
|
}
|
||||||
|
|
||||||
|
function AlertStatusTag({ status }: { status: string }) {
|
||||||
|
const cfg = STATUS_CONFIG[status] ?? { label: status, color: 'default', icon: null }
|
||||||
|
return (
|
||||||
|
<Tag color={cfg.color} icon={cfg.icon}>
|
||||||
|
{cfg.label}
|
||||||
|
</Tag>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AlertsPage() {
|
||||||
|
const navigate = useNavigate()
|
||||||
|
const qc = useQueryClient()
|
||||||
|
const [statusFilter, setStatusFilter] = useState<string>('open')
|
||||||
|
|
||||||
|
const { data: alerts, isLoading } = useAlerts({
|
||||||
|
status: statusFilter === 'all' ? undefined : statusFilter,
|
||||||
|
limit: 200,
|
||||||
|
refetchInterval: 60_000,
|
||||||
|
})
|
||||||
|
|
||||||
|
const acknowledge = useAcknowledgeAlert()
|
||||||
|
const resolve = useResolveAlert()
|
||||||
|
|
||||||
|
// Live updates via monitor SSE
|
||||||
|
useMonitorSSE({
|
||||||
|
onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
|
onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
|
})
|
||||||
|
|
||||||
|
const handleAcknowledge = async (id: number) => {
|
||||||
|
try {
|
||||||
|
await acknowledge.mutateAsync(id)
|
||||||
|
message.success('Алерт принят')
|
||||||
|
} catch {
|
||||||
|
message.error('Ошибка')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleResolve = async (id: number) => {
|
||||||
|
try {
|
||||||
|
await resolve.mutateAsync(id)
|
||||||
|
message.success('Алерт закрыт')
|
||||||
|
} catch {
|
||||||
|
message.error('Ошибка')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const columns = [
|
||||||
|
{
|
||||||
|
title: 'Статус',
|
||||||
|
dataIndex: 'status',
|
||||||
|
key: 'status',
|
||||||
|
width: 130,
|
||||||
|
render: (v: string) => <AlertStatusTag status={v} />,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Устройство',
|
||||||
|
key: 'device',
|
||||||
|
render: (_: unknown, row: AlertItem) => (
|
||||||
|
<Space direction="vertical" size={0}>
|
||||||
|
<code style={{ fontSize: 13 }}>{row.device_ip ?? '—'}</code>
|
||||||
|
{row.device_hostname && (
|
||||||
|
<Typography.Text type="secondary" style={{ fontSize: 12 }}>
|
||||||
|
{row.device_hostname}
|
||||||
|
</Typography.Text>
|
||||||
|
)}
|
||||||
|
</Space>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Объект',
|
||||||
|
dataIndex: 'object_name',
|
||||||
|
key: 'object_name',
|
||||||
|
render: (v: string | null, row: AlertItem) =>
|
||||||
|
v ? (
|
||||||
|
<a onClick={() => navigate(`/objects/${row.object_id}`)}>{v}</a>
|
||||||
|
) : (
|
||||||
|
'—'
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Сообщение',
|
||||||
|
dataIndex: 'message',
|
||||||
|
key: 'message',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Создан',
|
||||||
|
dataIndex: 'created_at',
|
||||||
|
key: 'created_at',
|
||||||
|
render: (v: string) => (
|
||||||
|
<Tooltip title={dayjs(v).format('DD.MM.YYYY HH:mm:ss')}>
|
||||||
|
{dayjs(v).format('DD.MM HH:mm')}
|
||||||
|
</Tooltip>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Принят',
|
||||||
|
key: 'acknowledged',
|
||||||
|
render: (_: unknown, row: AlertItem) =>
|
||||||
|
row.acknowledged_at ? (
|
||||||
|
<Space direction="vertical" size={0}>
|
||||||
|
<Typography.Text style={{ fontSize: 12 }}>
|
||||||
|
{dayjs(row.acknowledged_at).format('DD.MM HH:mm')}
|
||||||
|
</Typography.Text>
|
||||||
|
{row.acknowledged_by_username && (
|
||||||
|
<Typography.Text type="secondary" style={{ fontSize: 11 }}>
|
||||||
|
{row.acknowledged_by_username}
|
||||||
|
</Typography.Text>
|
||||||
|
)}
|
||||||
|
</Space>
|
||||||
|
) : (
|
||||||
|
'—'
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: '',
|
||||||
|
key: 'actions',
|
||||||
|
width: 160,
|
||||||
|
render: (_: unknown, row: AlertItem) => (
|
||||||
|
<Space>
|
||||||
|
{row.status === 'open' && (
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
icon={<ClockCircleOutlined />}
|
||||||
|
onClick={() => handleAcknowledge(row.id)}
|
||||||
|
loading={acknowledge.isPending}
|
||||||
|
>
|
||||||
|
Принять
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
{row.status !== 'resolved' && (
|
||||||
|
<Popconfirm
|
||||||
|
title="Закрыть алерт?"
|
||||||
|
okText="Закрыть"
|
||||||
|
cancelText="Отмена"
|
||||||
|
onConfirm={() => handleResolve(row.id)}
|
||||||
|
>
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
icon={<CheckCircleOutlined />}
|
||||||
|
loading={resolve.isPending}
|
||||||
|
>
|
||||||
|
Закрыть
|
||||||
|
</Button>
|
||||||
|
</Popconfirm>
|
||||||
|
)}
|
||||||
|
</Space>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
const openCount = (alerts ?? []).filter((a) => a.status === 'open').length
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<Breadcrumb
|
||||||
|
style={{ marginBottom: 16 }}
|
||||||
|
items={[{ title: 'Алерты' }]}
|
||||||
|
/>
|
||||||
|
|
||||||
|
<Row justify="space-between" align="middle" style={{ marginBottom: 16 }}>
|
||||||
|
<Col>
|
||||||
|
<Space align="center">
|
||||||
|
<Typography.Title level={4} style={{ margin: 0 }}>
|
||||||
|
Алерты
|
||||||
|
</Typography.Title>
|
||||||
|
{openCount > 0 && (
|
||||||
|
<Badge count={openCount} color="red" />
|
||||||
|
)}
|
||||||
|
</Space>
|
||||||
|
</Col>
|
||||||
|
<Col>
|
||||||
|
<Select
|
||||||
|
value={statusFilter}
|
||||||
|
onChange={setStatusFilter}
|
||||||
|
style={{ width: 160 }}
|
||||||
|
options={[
|
||||||
|
{ value: 'open', label: 'Открытые' },
|
||||||
|
{ value: 'acknowledged', label: 'Принятые' },
|
||||||
|
{ value: 'resolved', label: 'Закрытые' },
|
||||||
|
{ value: 'all', label: 'Все' },
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
<Table
|
||||||
|
dataSource={alerts ?? []}
|
||||||
|
columns={columns}
|
||||||
|
rowKey="id"
|
||||||
|
loading={isLoading}
|
||||||
|
size="middle"
|
||||||
|
pagination={{ pageSize: 50 }}
|
||||||
|
rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -12,6 +12,7 @@ import {
|
||||||
Alert,
|
Alert,
|
||||||
Breadcrumb,
|
Breadcrumb,
|
||||||
Button,
|
Button,
|
||||||
|
Checkbox,
|
||||||
Col,
|
Col,
|
||||||
Descriptions,
|
Descriptions,
|
||||||
Divider,
|
Divider,
|
||||||
|
|
@ -145,6 +146,7 @@ export function ObjectDetailPage() {
|
||||||
model: row.model ?? '',
|
model: row.model ?? '',
|
||||||
mac: row.mac ?? '',
|
mac: row.mac ?? '',
|
||||||
rack_id: row.rack_id ?? undefined,
|
rack_id: row.rack_id ?? undefined,
|
||||||
|
monitor_enabled: row.monitor_enabled ?? false,
|
||||||
ssh_user_override: row.ssh_user_override ?? '',
|
ssh_user_override: row.ssh_user_override ?? '',
|
||||||
ssh_port_override: row.ssh_port_override ?? undefined,
|
ssh_port_override: row.ssh_port_override ?? undefined,
|
||||||
})
|
})
|
||||||
|
|
@ -758,6 +760,10 @@ export function ObjectDetailPage() {
|
||||||
</Col>
|
</Col>
|
||||||
</Row>
|
</Row>
|
||||||
|
|
||||||
|
<Form.Item name="monitor_enabled" valuePropName="checked">
|
||||||
|
<Checkbox>Мониторинг (автоматический ping)</Checkbox>
|
||||||
|
</Form.Item>
|
||||||
|
|
||||||
<Divider orientation="left" plain>
|
<Divider orientation="left" plain>
|
||||||
SSH (переопределить настройки объекта)
|
SSH (переопределить настройки объекта)
|
||||||
</Divider>
|
</Divider>
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue