фикс алертов
This commit is contained in:
parent
48b1b737f3
commit
e8a712aee4
4 changed files with 224 additions and 25 deletions
|
|
@ -91,6 +91,42 @@ async def list_alerts(
|
|||
return [_to_read(a) for a in result.scalars().all()]
|
||||
|
||||
|
||||
@router.post("/acknowledge-by-object", response_model=list[AlertRead])
|
||||
async def acknowledge_by_object(
|
||||
object_id: int = Query(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
result = await db.execute(
|
||||
select(Alert)
|
||||
.join(Device, Alert.device_id == Device.id)
|
||||
.where(Device.object_id == object_id, Alert.status == "open")
|
||||
)
|
||||
alerts = result.scalars().all()
|
||||
now = datetime.now(timezone.utc)
|
||||
for alert in alerts:
|
||||
alert.status = "acknowledged"
|
||||
alert.acknowledged_at = now
|
||||
alert.acknowledged_by = current_user.id
|
||||
await db.flush()
|
||||
|
||||
alert_ids = [a.id for a in alerts]
|
||||
loaded = []
|
||||
for aid in alert_ids:
|
||||
loaded.append(await _load_alert(db, aid))
|
||||
|
||||
for alert in loaded:
|
||||
await sse_manager.publish(
|
||||
SSEEvent(
|
||||
type="alert_updated",
|
||||
task_id=MONITOR_CHANNEL,
|
||||
data={"alert_id": alert.id, "status": "acknowledged", "device_id": alert.device_id},
|
||||
)
|
||||
)
|
||||
|
||||
return [_to_read(a) for a in loaded]
|
||||
|
||||
|
||||
@router.post("/{alert_id}/acknowledge", response_model=AlertRead)
|
||||
async def acknowledge_alert(
|
||||
alert_id: int,
|
||||
|
|
|
|||
|
|
@ -549,6 +549,19 @@ async def run_dashboard_monitor_cycle(
|
|||
", ".join(ips[:10]) + ("…" if len(ips) > 10 else ""),
|
||||
)
|
||||
|
||||
# Загружаем предыдущие ping-статусы до запуска — нужны для retry-логики
|
||||
pre_ping_status_map: dict[int, str] = {}
|
||||
if check.check_key == "ping":
|
||||
pre_state_rows = (
|
||||
await db.execute(
|
||||
select(DashboardDeviceLiveState).where(
|
||||
DashboardDeviceLiveState.check_key == "ping",
|
||||
DashboardDeviceLiveState.device_id.in_([d.id for d in due_devices]),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
pre_ping_status_map = {row.device_id: row.status for row in pre_state_rows}
|
||||
|
||||
sem = asyncio.Semaphore(max(1, check.concurrency))
|
||||
tasks = [
|
||||
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
|
||||
|
|
@ -589,6 +602,78 @@ async def run_dashboard_monitor_cycle(
|
|||
)
|
||||
)
|
||||
|
||||
# Retry для offline-устройств (только ping):
|
||||
# срабатывает только если устройство было online (переход online→offline)
|
||||
if check.check_key == "ping":
|
||||
offline_devices = [
|
||||
r[0] for r in results
|
||||
if r[2] == "offline" and pre_ping_status_map.get(r[0].id) == "online"
|
||||
]
|
||||
for retry_num in range(1, 3):
|
||||
if not offline_devices:
|
||||
break
|
||||
logger.info(
|
||||
"Dashboard monitor [ping] retry %d/2: %d offline устройств, ждём 60 сек",
|
||||
retry_num, len(offline_devices),
|
||||
)
|
||||
await sse_manager.publish(
|
||||
SSEEvent(
|
||||
type="dashboard_monitor_update",
|
||||
task_id=DASHBOARD_MONITOR_CHANNEL,
|
||||
data={
|
||||
"check_key": check.check_key,
|
||||
"processed": completed_count,
|
||||
"due": len(due_devices),
|
||||
"failed": failed_count,
|
||||
"retry": retry_num,
|
||||
"retry_pending": len(offline_devices),
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(60)
|
||||
|
||||
retry_tasks = [
|
||||
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
|
||||
for device in offline_devices
|
||||
]
|
||||
retry_results = []
|
||||
for coro in asyncio.as_completed(retry_tasks):
|
||||
r = await coro
|
||||
retry_results.append(r)
|
||||
logger.debug(
|
||||
"Dashboard monitor [ping] retry %d %s: %s",
|
||||
retry_num, r[0].ip, r[2],
|
||||
)
|
||||
|
||||
# Обновляем results: заменяем результат устройства если оно восстановилось
|
||||
retry_map = {r[0].id: r for r in retry_results}
|
||||
results = [
|
||||
retry_map.get(r[0].id, r) for r in results
|
||||
]
|
||||
offline_devices = [r[0] for r in retry_results if r[2] == "offline"]
|
||||
recovered = len(retry_results) - len(offline_devices)
|
||||
failed_count = sum(1 for r in results if not r[1])
|
||||
logger.info(
|
||||
"Dashboard monitor [ping] retry %d/2 done: восстановилось %d, всё ещё offline %d",
|
||||
retry_num, recovered, len(offline_devices),
|
||||
)
|
||||
await sse_manager.publish(
|
||||
SSEEvent(
|
||||
type="dashboard_monitor_update",
|
||||
task_id=DASHBOARD_MONITOR_CHANNEL,
|
||||
data={
|
||||
"check_key": check.check_key,
|
||||
"processed": len(due_devices),
|
||||
"due": len(due_devices),
|
||||
"failed": failed_count,
|
||||
"retry": retry_num,
|
||||
"retry_recovered": recovered,
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
async with AsyncSessionLocal() as dbw:
|
||||
async with dbw.begin():
|
||||
# Загружаем предыдущие статусы и baseline только для ping
|
||||
|
|
|
|||
|
|
@ -50,3 +50,16 @@ export const useResolveAlert = () => {
|
|||
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||
})
|
||||
}
|
||||
|
||||
export const useAcknowledgeByObject = () => {
|
||||
const qc = useQueryClient()
|
||||
return useMutation({
|
||||
mutationFn: (objectId: number) =>
|
||||
api
|
||||
.post<AlertItem[]>(`/api/v1/alerts/acknowledge-by-object`, null, {
|
||||
params: { object_id: objectId },
|
||||
})
|
||||
.then((r) => r.data),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,13 +18,14 @@ import {
|
|||
Typography,
|
||||
message,
|
||||
} from 'antd'
|
||||
import { useState } from 'react'
|
||||
import { useState, useMemo } from 'react'
|
||||
import { useQueryClient } from '@tanstack/react-query'
|
||||
import { useNavigate } from 'react-router-dom'
|
||||
import {
|
||||
useAlerts,
|
||||
useAcknowledgeAlert,
|
||||
useResolveAlert,
|
||||
useAcknowledgeByObject,
|
||||
type AlertItem,
|
||||
} from '../api/alerts'
|
||||
import { useMonitorSSE } from '../hooks/useMonitorSSE'
|
||||
|
|
@ -48,6 +49,12 @@ function AlertStatusTag({ status }: { status: string }) {
|
|||
)
|
||||
}
|
||||
|
||||
interface ObjectGroup {
|
||||
object_id: number | null
|
||||
object_name: string | null
|
||||
alerts: AlertItem[]
|
||||
}
|
||||
|
||||
export function AlertsPage() {
|
||||
const navigate = useNavigate()
|
||||
const qc = useQueryClient()
|
||||
|
|
@ -61,13 +68,28 @@ export function AlertsPage() {
|
|||
|
||||
const acknowledge = useAcknowledgeAlert()
|
||||
const resolve = useResolveAlert()
|
||||
const acknowledgeByObject = useAcknowledgeByObject()
|
||||
|
||||
// Live updates via monitor SSE
|
||||
useMonitorSSE({
|
||||
onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||
onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||
})
|
||||
|
||||
const groups = useMemo<ObjectGroup[]>(() => {
|
||||
if (!alerts) return []
|
||||
const map = new Map<number | null, ObjectGroup>()
|
||||
for (const a of alerts) {
|
||||
const key = a.object_id ?? null
|
||||
if (!map.has(key)) {
|
||||
map.set(key, { object_id: key, object_name: a.object_name, alerts: [] })
|
||||
}
|
||||
map.get(key)!.alerts.push(a)
|
||||
}
|
||||
return Array.from(map.values()).sort((a, b) =>
|
||||
(a.object_name ?? '').localeCompare(b.object_name ?? '', 'ru')
|
||||
)
|
||||
}, [alerts])
|
||||
|
||||
const handleAcknowledge = async (id: number) => {
|
||||
try {
|
||||
await acknowledge.mutateAsync(id)
|
||||
|
|
@ -86,6 +108,15 @@ export function AlertsPage() {
|
|||
}
|
||||
}
|
||||
|
||||
const handleAcknowledgeByObject = async (objectId: number) => {
|
||||
try {
|
||||
await acknowledgeByObject.mutateAsync(objectId)
|
||||
message.success('Все алерты объекта приняты')
|
||||
} catch {
|
||||
message.error('Ошибка')
|
||||
}
|
||||
}
|
||||
|
||||
const columns = [
|
||||
{
|
||||
title: 'Статус',
|
||||
|
|
@ -108,17 +139,6 @@ export function AlertsPage() {
|
|||
</Space>
|
||||
),
|
||||
},
|
||||
{
|
||||
title: 'Объект',
|
||||
dataIndex: 'object_name',
|
||||
key: 'object_name',
|
||||
render: (v: string | null, row: AlertItem) =>
|
||||
v ? (
|
||||
<a onClick={() => navigate(`/objects/${row.object_id}`)}>{v}</a>
|
||||
) : (
|
||||
'—'
|
||||
),
|
||||
},
|
||||
{
|
||||
title: 'Сообщение',
|
||||
dataIndex: 'message',
|
||||
|
|
@ -128,6 +148,7 @@ export function AlertsPage() {
|
|||
title: 'Создан',
|
||||
dataIndex: 'created_at',
|
||||
key: 'created_at',
|
||||
width: 110,
|
||||
render: (v: string) => (
|
||||
<Tooltip title={dayjs(v).format('DD.MM.YYYY HH:mm:ss')}>
|
||||
{dayjs(v).format('DD.MM HH:mm')}
|
||||
|
|
@ -137,6 +158,7 @@ export function AlertsPage() {
|
|||
{
|
||||
title: 'Принят',
|
||||
key: 'acknowledged',
|
||||
width: 130,
|
||||
render: (_: unknown, row: AlertItem) =>
|
||||
row.acknowledged_at ? (
|
||||
<Space direction="vertical" size={0}>
|
||||
|
|
@ -205,9 +227,7 @@ export function AlertsPage() {
|
|||
<Typography.Title level={4} style={{ margin: 0 }}>
|
||||
Алерты
|
||||
</Typography.Title>
|
||||
{openCount > 0 && (
|
||||
<Badge count={openCount} color="red" />
|
||||
)}
|
||||
{openCount > 0 && <Badge count={openCount} color="red" />}
|
||||
</Space>
|
||||
</Col>
|
||||
<Col>
|
||||
|
|
@ -225,15 +245,60 @@ export function AlertsPage() {
|
|||
</Col>
|
||||
</Row>
|
||||
|
||||
{isLoading ? (
|
||||
<Table loading columns={columns} dataSource={[]} rowKey="id" />
|
||||
) : (
|
||||
groups.map((group) => {
|
||||
const openInGroup = group.alerts.filter((a) => a.status === 'open').length
|
||||
return (
|
||||
<div key={group.object_id ?? 'unknown'} style={{ marginBottom: 24 }}>
|
||||
<Row justify="space-between" align="middle" style={{ marginBottom: 8 }}>
|
||||
<Col>
|
||||
<Space align="center">
|
||||
{group.object_id ? (
|
||||
<Typography.Link
|
||||
strong
|
||||
onClick={() => navigate(`/work/objects/${group.object_id}`)}
|
||||
>
|
||||
{group.object_name ?? `Объект #${group.object_id}`}
|
||||
</Typography.Link>
|
||||
) : (
|
||||
<Typography.Text strong>Без объекта</Typography.Text>
|
||||
)}
|
||||
<Badge count={group.alerts.length} color="blue" />
|
||||
</Space>
|
||||
</Col>
|
||||
{openInGroup > 0 && group.object_id != null && (
|
||||
<Col>
|
||||
<Popconfirm
|
||||
title={`Принять все ${openInGroup} открытых алертов объекта?`}
|
||||
okText="Принять все"
|
||||
cancelText="Отмена"
|
||||
onConfirm={() => handleAcknowledgeByObject(group.object_id!)}
|
||||
>
|
||||
<Button
|
||||
size="small"
|
||||
icon={<ClockCircleOutlined />}
|
||||
loading={acknowledgeByObject.isPending}
|
||||
>
|
||||
Принять все ({openInGroup})
|
||||
</Button>
|
||||
</Popconfirm>
|
||||
</Col>
|
||||
)}
|
||||
</Row>
|
||||
<Table
|
||||
dataSource={alerts ?? []}
|
||||
dataSource={group.alerts}
|
||||
columns={columns}
|
||||
rowKey="id"
|
||||
loading={isLoading}
|
||||
size="middle"
|
||||
pagination={{ pageSize: 50 }}
|
||||
size="small"
|
||||
pagination={false}
|
||||
rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue