фикс алертов

This commit is contained in:
dv 2026-05-06 15:05:00 +03:00
parent 48b1b737f3
commit e8a712aee4
4 changed files with 224 additions and 25 deletions

View file

@ -91,6 +91,42 @@ async def list_alerts(
return [_to_read(a) for a in result.scalars().all()] return [_to_read(a) for a in result.scalars().all()]
@router.post("/acknowledge-by-object", response_model=list[AlertRead])
async def acknowledge_by_object(
object_id: int = Query(...),
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user),
):
result = await db.execute(
select(Alert)
.join(Device, Alert.device_id == Device.id)
.where(Device.object_id == object_id, Alert.status == "open")
)
alerts = result.scalars().all()
now = datetime.now(timezone.utc)
for alert in alerts:
alert.status = "acknowledged"
alert.acknowledged_at = now
alert.acknowledged_by = current_user.id
await db.flush()
alert_ids = [a.id for a in alerts]
loaded = []
for aid in alert_ids:
loaded.append(await _load_alert(db, aid))
for alert in loaded:
await sse_manager.publish(
SSEEvent(
type="alert_updated",
task_id=MONITOR_CHANNEL,
data={"alert_id": alert.id, "status": "acknowledged", "device_id": alert.device_id},
)
)
return [_to_read(a) for a in loaded]
@router.post("/{alert_id}/acknowledge", response_model=AlertRead) @router.post("/{alert_id}/acknowledge", response_model=AlertRead)
async def acknowledge_alert( async def acknowledge_alert(
alert_id: int, alert_id: int,

View file

@ -549,6 +549,19 @@ async def run_dashboard_monitor_cycle(
", ".join(ips[:10]) + ("" if len(ips) > 10 else ""), ", ".join(ips[:10]) + ("" if len(ips) > 10 else ""),
) )
# Загружаем предыдущие ping-статусы до запуска — нужны для retry-логики
pre_ping_status_map: dict[int, str] = {}
if check.check_key == "ping":
pre_state_rows = (
await db.execute(
select(DashboardDeviceLiveState).where(
DashboardDeviceLiveState.check_key == "ping",
DashboardDeviceLiveState.device_id.in_([d.id for d in due_devices]),
)
)
).scalars().all()
pre_ping_status_map = {row.device_id: row.status for row in pre_state_rows}
sem = asyncio.Semaphore(max(1, check.concurrency)) sem = asyncio.Semaphore(max(1, check.concurrency))
tasks = [ tasks = [
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object)) asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
@ -589,6 +602,78 @@ async def run_dashboard_monitor_cycle(
) )
) )
# Retry для offline-устройств (только ping):
# срабатывает только если устройство было online (переход online→offline)
if check.check_key == "ping":
offline_devices = [
r[0] for r in results
if r[2] == "offline" and pre_ping_status_map.get(r[0].id) == "online"
]
for retry_num in range(1, 3):
if not offline_devices:
break
logger.info(
"Dashboard monitor [ping] retry %d/2: %d offline устройств, ждём 60 сек",
retry_num, len(offline_devices),
)
await sse_manager.publish(
SSEEvent(
type="dashboard_monitor_update",
task_id=DASHBOARD_MONITOR_CHANNEL,
data={
"check_key": check.check_key,
"processed": completed_count,
"due": len(due_devices),
"failed": failed_count,
"retry": retry_num,
"retry_pending": len(offline_devices),
"ts": datetime.now(timezone.utc).isoformat(),
},
)
)
await asyncio.sleep(60)
retry_tasks = [
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
for device in offline_devices
]
retry_results = []
for coro in asyncio.as_completed(retry_tasks):
r = await coro
retry_results.append(r)
logger.debug(
"Dashboard monitor [ping] retry %d %s: %s",
retry_num, r[0].ip, r[2],
)
# Обновляем results: заменяем результат устройства если оно восстановилось
retry_map = {r[0].id: r for r in retry_results}
results = [
retry_map.get(r[0].id, r) for r in results
]
offline_devices = [r[0] for r in retry_results if r[2] == "offline"]
recovered = len(retry_results) - len(offline_devices)
failed_count = sum(1 for r in results if not r[1])
logger.info(
"Dashboard monitor [ping] retry %d/2 done: восстановилось %d, всё ещё offline %d",
retry_num, recovered, len(offline_devices),
)
await sse_manager.publish(
SSEEvent(
type="dashboard_monitor_update",
task_id=DASHBOARD_MONITOR_CHANNEL,
data={
"check_key": check.check_key,
"processed": len(due_devices),
"due": len(due_devices),
"failed": failed_count,
"retry": retry_num,
"retry_recovered": recovered,
"ts": datetime.now(timezone.utc).isoformat(),
},
)
)
async with AsyncSessionLocal() as dbw: async with AsyncSessionLocal() as dbw:
async with dbw.begin(): async with dbw.begin():
# Загружаем предыдущие статусы и baseline только для ping # Загружаем предыдущие статусы и baseline только для ping

View file

@ -50,3 +50,16 @@ export const useResolveAlert = () => {
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }), onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
}) })
} }
export const useAcknowledgeByObject = () => {
const qc = useQueryClient()
return useMutation({
mutationFn: (objectId: number) =>
api
.post<AlertItem[]>(`/api/v1/alerts/acknowledge-by-object`, null, {
params: { object_id: objectId },
})
.then((r) => r.data),
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
})
}

View file

@ -18,13 +18,14 @@ import {
Typography, Typography,
message, message,
} from 'antd' } from 'antd'
import { useState } from 'react' import { useState, useMemo } from 'react'
import { useQueryClient } from '@tanstack/react-query' import { useQueryClient } from '@tanstack/react-query'
import { useNavigate } from 'react-router-dom' import { useNavigate } from 'react-router-dom'
import { import {
useAlerts, useAlerts,
useAcknowledgeAlert, useAcknowledgeAlert,
useResolveAlert, useResolveAlert,
useAcknowledgeByObject,
type AlertItem, type AlertItem,
} from '../api/alerts' } from '../api/alerts'
import { useMonitorSSE } from '../hooks/useMonitorSSE' import { useMonitorSSE } from '../hooks/useMonitorSSE'
@ -48,6 +49,12 @@ function AlertStatusTag({ status }: { status: string }) {
) )
} }
interface ObjectGroup {
object_id: number | null
object_name: string | null
alerts: AlertItem[]
}
export function AlertsPage() { export function AlertsPage() {
const navigate = useNavigate() const navigate = useNavigate()
const qc = useQueryClient() const qc = useQueryClient()
@ -61,13 +68,28 @@ export function AlertsPage() {
const acknowledge = useAcknowledgeAlert() const acknowledge = useAcknowledgeAlert()
const resolve = useResolveAlert() const resolve = useResolveAlert()
const acknowledgeByObject = useAcknowledgeByObject()
// Live updates via monitor SSE
useMonitorSSE({ useMonitorSSE({
onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }), onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }), onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
}) })
const groups = useMemo<ObjectGroup[]>(() => {
if (!alerts) return []
const map = new Map<number | null, ObjectGroup>()
for (const a of alerts) {
const key = a.object_id ?? null
if (!map.has(key)) {
map.set(key, { object_id: key, object_name: a.object_name, alerts: [] })
}
map.get(key)!.alerts.push(a)
}
return Array.from(map.values()).sort((a, b) =>
(a.object_name ?? '').localeCompare(b.object_name ?? '', 'ru')
)
}, [alerts])
const handleAcknowledge = async (id: number) => { const handleAcknowledge = async (id: number) => {
try { try {
await acknowledge.mutateAsync(id) await acknowledge.mutateAsync(id)
@ -86,6 +108,15 @@ export function AlertsPage() {
} }
} }
const handleAcknowledgeByObject = async (objectId: number) => {
try {
await acknowledgeByObject.mutateAsync(objectId)
message.success('Все алерты объекта приняты')
} catch {
message.error('Ошибка')
}
}
const columns = [ const columns = [
{ {
title: 'Статус', title: 'Статус',
@ -108,17 +139,6 @@ export function AlertsPage() {
</Space> </Space>
), ),
}, },
{
title: 'Объект',
dataIndex: 'object_name',
key: 'object_name',
render: (v: string | null, row: AlertItem) =>
v ? (
<a onClick={() => navigate(`/objects/${row.object_id}`)}>{v}</a>
) : (
'—'
),
},
{ {
title: 'Сообщение', title: 'Сообщение',
dataIndex: 'message', dataIndex: 'message',
@ -128,6 +148,7 @@ export function AlertsPage() {
title: 'Создан', title: 'Создан',
dataIndex: 'created_at', dataIndex: 'created_at',
key: 'created_at', key: 'created_at',
width: 110,
render: (v: string) => ( render: (v: string) => (
<Tooltip title={dayjs(v).format('DD.MM.YYYY HH:mm:ss')}> <Tooltip title={dayjs(v).format('DD.MM.YYYY HH:mm:ss')}>
{dayjs(v).format('DD.MM HH:mm')} {dayjs(v).format('DD.MM HH:mm')}
@ -137,6 +158,7 @@ export function AlertsPage() {
{ {
title: 'Принят', title: 'Принят',
key: 'acknowledged', key: 'acknowledged',
width: 130,
render: (_: unknown, row: AlertItem) => render: (_: unknown, row: AlertItem) =>
row.acknowledged_at ? ( row.acknowledged_at ? (
<Space direction="vertical" size={0}> <Space direction="vertical" size={0}>
@ -205,9 +227,7 @@ export function AlertsPage() {
<Typography.Title level={4} style={{ margin: 0 }}> <Typography.Title level={4} style={{ margin: 0 }}>
Алерты Алерты
</Typography.Title> </Typography.Title>
{openCount > 0 && ( {openCount > 0 && <Badge count={openCount} color="red" />}
<Badge count={openCount} color="red" />
)}
</Space> </Space>
</Col> </Col>
<Col> <Col>
@ -225,15 +245,60 @@ export function AlertsPage() {
</Col> </Col>
</Row> </Row>
{isLoading ? (
<Table loading columns={columns} dataSource={[]} rowKey="id" />
) : (
groups.map((group) => {
const openInGroup = group.alerts.filter((a) => a.status === 'open').length
return (
<div key={group.object_id ?? 'unknown'} style={{ marginBottom: 24 }}>
<Row justify="space-between" align="middle" style={{ marginBottom: 8 }}>
<Col>
<Space align="center">
{group.object_id ? (
<Typography.Link
strong
onClick={() => navigate(`/work/objects/${group.object_id}`)}
>
{group.object_name ?? `Объект #${group.object_id}`}
</Typography.Link>
) : (
<Typography.Text strong>Без объекта</Typography.Text>
)}
<Badge count={group.alerts.length} color="blue" />
</Space>
</Col>
{openInGroup > 0 && group.object_id != null && (
<Col>
<Popconfirm
title={`Принять все ${openInGroup} открытых алертов объекта?`}
okText="Принять все"
cancelText="Отмена"
onConfirm={() => handleAcknowledgeByObject(group.object_id!)}
>
<Button
size="small"
icon={<ClockCircleOutlined />}
loading={acknowledgeByObject.isPending}
>
Принять все ({openInGroup})
</Button>
</Popconfirm>
</Col>
)}
</Row>
<Table <Table
dataSource={alerts ?? []} dataSource={group.alerts}
columns={columns} columns={columns}
rowKey="id" rowKey="id"
loading={isLoading} size="small"
size="middle" pagination={false}
pagination={{ pageSize: 50 }}
rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')} rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')}
/> />
</div> </div>
) )
})
)}
</div>
)
} }