фикс алертов
This commit is contained in:
parent
48b1b737f3
commit
e8a712aee4
4 changed files with 224 additions and 25 deletions
|
|
@ -91,6 +91,42 @@ async def list_alerts(
|
||||||
return [_to_read(a) for a in result.scalars().all()]
|
return [_to_read(a) for a in result.scalars().all()]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/acknowledge-by-object", response_model=list[AlertRead])
|
||||||
|
async def acknowledge_by_object(
|
||||||
|
object_id: int = Query(...),
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
result = await db.execute(
|
||||||
|
select(Alert)
|
||||||
|
.join(Device, Alert.device_id == Device.id)
|
||||||
|
.where(Device.object_id == object_id, Alert.status == "open")
|
||||||
|
)
|
||||||
|
alerts = result.scalars().all()
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
for alert in alerts:
|
||||||
|
alert.status = "acknowledged"
|
||||||
|
alert.acknowledged_at = now
|
||||||
|
alert.acknowledged_by = current_user.id
|
||||||
|
await db.flush()
|
||||||
|
|
||||||
|
alert_ids = [a.id for a in alerts]
|
||||||
|
loaded = []
|
||||||
|
for aid in alert_ids:
|
||||||
|
loaded.append(await _load_alert(db, aid))
|
||||||
|
|
||||||
|
for alert in loaded:
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="alert_updated",
|
||||||
|
task_id=MONITOR_CHANNEL,
|
||||||
|
data={"alert_id": alert.id, "status": "acknowledged", "device_id": alert.device_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return [_to_read(a) for a in loaded]
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{alert_id}/acknowledge", response_model=AlertRead)
|
@router.post("/{alert_id}/acknowledge", response_model=AlertRead)
|
||||||
async def acknowledge_alert(
|
async def acknowledge_alert(
|
||||||
alert_id: int,
|
alert_id: int,
|
||||||
|
|
|
||||||
|
|
@ -549,6 +549,19 @@ async def run_dashboard_monitor_cycle(
|
||||||
", ".join(ips[:10]) + ("…" if len(ips) > 10 else ""),
|
", ".join(ips[:10]) + ("…" if len(ips) > 10 else ""),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Загружаем предыдущие ping-статусы до запуска — нужны для retry-логики
|
||||||
|
pre_ping_status_map: dict[int, str] = {}
|
||||||
|
if check.check_key == "ping":
|
||||||
|
pre_state_rows = (
|
||||||
|
await db.execute(
|
||||||
|
select(DashboardDeviceLiveState).where(
|
||||||
|
DashboardDeviceLiveState.check_key == "ping",
|
||||||
|
DashboardDeviceLiveState.device_id.in_([d.id for d in due_devices]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
pre_ping_status_map = {row.device_id: row.status for row in pre_state_rows}
|
||||||
|
|
||||||
sem = asyncio.Semaphore(max(1, check.concurrency))
|
sem = asyncio.Semaphore(max(1, check.concurrency))
|
||||||
tasks = [
|
tasks = [
|
||||||
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
|
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
|
||||||
|
|
@ -589,6 +602,78 @@ async def run_dashboard_monitor_cycle(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Retry для offline-устройств (только ping):
|
||||||
|
# срабатывает только если устройство было online (переход online→offline)
|
||||||
|
if check.check_key == "ping":
|
||||||
|
offline_devices = [
|
||||||
|
r[0] for r in results
|
||||||
|
if r[2] == "offline" and pre_ping_status_map.get(r[0].id) == "online"
|
||||||
|
]
|
||||||
|
for retry_num in range(1, 3):
|
||||||
|
if not offline_devices:
|
||||||
|
break
|
||||||
|
logger.info(
|
||||||
|
"Dashboard monitor [ping] retry %d/2: %d offline устройств, ждём 60 сек",
|
||||||
|
retry_num, len(offline_devices),
|
||||||
|
)
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="dashboard_monitor_update",
|
||||||
|
task_id=DASHBOARD_MONITOR_CHANNEL,
|
||||||
|
data={
|
||||||
|
"check_key": check.check_key,
|
||||||
|
"processed": completed_count,
|
||||||
|
"due": len(due_devices),
|
||||||
|
"failed": failed_count,
|
||||||
|
"retry": retry_num,
|
||||||
|
"retry_pending": len(offline_devices),
|
||||||
|
"ts": datetime.now(timezone.utc).isoformat(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await asyncio.sleep(60)
|
||||||
|
|
||||||
|
retry_tasks = [
|
||||||
|
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
|
||||||
|
for device in offline_devices
|
||||||
|
]
|
||||||
|
retry_results = []
|
||||||
|
for coro in asyncio.as_completed(retry_tasks):
|
||||||
|
r = await coro
|
||||||
|
retry_results.append(r)
|
||||||
|
logger.debug(
|
||||||
|
"Dashboard monitor [ping] retry %d %s: %s",
|
||||||
|
retry_num, r[0].ip, r[2],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Обновляем results: заменяем результат устройства если оно восстановилось
|
||||||
|
retry_map = {r[0].id: r for r in retry_results}
|
||||||
|
results = [
|
||||||
|
retry_map.get(r[0].id, r) for r in results
|
||||||
|
]
|
||||||
|
offline_devices = [r[0] for r in retry_results if r[2] == "offline"]
|
||||||
|
recovered = len(retry_results) - len(offline_devices)
|
||||||
|
failed_count = sum(1 for r in results if not r[1])
|
||||||
|
logger.info(
|
||||||
|
"Dashboard monitor [ping] retry %d/2 done: восстановилось %d, всё ещё offline %d",
|
||||||
|
retry_num, recovered, len(offline_devices),
|
||||||
|
)
|
||||||
|
await sse_manager.publish(
|
||||||
|
SSEEvent(
|
||||||
|
type="dashboard_monitor_update",
|
||||||
|
task_id=DASHBOARD_MONITOR_CHANNEL,
|
||||||
|
data={
|
||||||
|
"check_key": check.check_key,
|
||||||
|
"processed": len(due_devices),
|
||||||
|
"due": len(due_devices),
|
||||||
|
"failed": failed_count,
|
||||||
|
"retry": retry_num,
|
||||||
|
"retry_recovered": recovered,
|
||||||
|
"ts": datetime.now(timezone.utc).isoformat(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
async with AsyncSessionLocal() as dbw:
|
async with AsyncSessionLocal() as dbw:
|
||||||
async with dbw.begin():
|
async with dbw.begin():
|
||||||
# Загружаем предыдущие статусы и baseline только для ping
|
# Загружаем предыдущие статусы и baseline только для ping
|
||||||
|
|
|
||||||
|
|
@ -50,3 +50,16 @@ export const useResolveAlert = () => {
|
||||||
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const useAcknowledgeByObject = () => {
|
||||||
|
const qc = useQueryClient()
|
||||||
|
return useMutation({
|
||||||
|
mutationFn: (objectId: number) =>
|
||||||
|
api
|
||||||
|
.post<AlertItem[]>(`/api/v1/alerts/acknowledge-by-object`, null, {
|
||||||
|
params: { object_id: objectId },
|
||||||
|
})
|
||||||
|
.then((r) => r.data),
|
||||||
|
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,13 +18,14 @@ import {
|
||||||
Typography,
|
Typography,
|
||||||
message,
|
message,
|
||||||
} from 'antd'
|
} from 'antd'
|
||||||
import { useState } from 'react'
|
import { useState, useMemo } from 'react'
|
||||||
import { useQueryClient } from '@tanstack/react-query'
|
import { useQueryClient } from '@tanstack/react-query'
|
||||||
import { useNavigate } from 'react-router-dom'
|
import { useNavigate } from 'react-router-dom'
|
||||||
import {
|
import {
|
||||||
useAlerts,
|
useAlerts,
|
||||||
useAcknowledgeAlert,
|
useAcknowledgeAlert,
|
||||||
useResolveAlert,
|
useResolveAlert,
|
||||||
|
useAcknowledgeByObject,
|
||||||
type AlertItem,
|
type AlertItem,
|
||||||
} from '../api/alerts'
|
} from '../api/alerts'
|
||||||
import { useMonitorSSE } from '../hooks/useMonitorSSE'
|
import { useMonitorSSE } from '../hooks/useMonitorSSE'
|
||||||
|
|
@ -48,6 +49,12 @@ function AlertStatusTag({ status }: { status: string }) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface ObjectGroup {
|
||||||
|
object_id: number | null
|
||||||
|
object_name: string | null
|
||||||
|
alerts: AlertItem[]
|
||||||
|
}
|
||||||
|
|
||||||
export function AlertsPage() {
|
export function AlertsPage() {
|
||||||
const navigate = useNavigate()
|
const navigate = useNavigate()
|
||||||
const qc = useQueryClient()
|
const qc = useQueryClient()
|
||||||
|
|
@ -61,13 +68,28 @@ export function AlertsPage() {
|
||||||
|
|
||||||
const acknowledge = useAcknowledgeAlert()
|
const acknowledge = useAcknowledgeAlert()
|
||||||
const resolve = useResolveAlert()
|
const resolve = useResolveAlert()
|
||||||
|
const acknowledgeByObject = useAcknowledgeByObject()
|
||||||
|
|
||||||
// Live updates via monitor SSE
|
|
||||||
useMonitorSSE({
|
useMonitorSSE({
|
||||||
onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const groups = useMemo<ObjectGroup[]>(() => {
|
||||||
|
if (!alerts) return []
|
||||||
|
const map = new Map<number | null, ObjectGroup>()
|
||||||
|
for (const a of alerts) {
|
||||||
|
const key = a.object_id ?? null
|
||||||
|
if (!map.has(key)) {
|
||||||
|
map.set(key, { object_id: key, object_name: a.object_name, alerts: [] })
|
||||||
|
}
|
||||||
|
map.get(key)!.alerts.push(a)
|
||||||
|
}
|
||||||
|
return Array.from(map.values()).sort((a, b) =>
|
||||||
|
(a.object_name ?? '').localeCompare(b.object_name ?? '', 'ru')
|
||||||
|
)
|
||||||
|
}, [alerts])
|
||||||
|
|
||||||
const handleAcknowledge = async (id: number) => {
|
const handleAcknowledge = async (id: number) => {
|
||||||
try {
|
try {
|
||||||
await acknowledge.mutateAsync(id)
|
await acknowledge.mutateAsync(id)
|
||||||
|
|
@ -86,6 +108,15 @@ export function AlertsPage() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const handleAcknowledgeByObject = async (objectId: number) => {
|
||||||
|
try {
|
||||||
|
await acknowledgeByObject.mutateAsync(objectId)
|
||||||
|
message.success('Все алерты объекта приняты')
|
||||||
|
} catch {
|
||||||
|
message.error('Ошибка')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const columns = [
|
const columns = [
|
||||||
{
|
{
|
||||||
title: 'Статус',
|
title: 'Статус',
|
||||||
|
|
@ -108,17 +139,6 @@ export function AlertsPage() {
|
||||||
</Space>
|
</Space>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
{
|
|
||||||
title: 'Объект',
|
|
||||||
dataIndex: 'object_name',
|
|
||||||
key: 'object_name',
|
|
||||||
render: (v: string | null, row: AlertItem) =>
|
|
||||||
v ? (
|
|
||||||
<a onClick={() => navigate(`/objects/${row.object_id}`)}>{v}</a>
|
|
||||||
) : (
|
|
||||||
'—'
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
title: 'Сообщение',
|
title: 'Сообщение',
|
||||||
dataIndex: 'message',
|
dataIndex: 'message',
|
||||||
|
|
@ -128,6 +148,7 @@ export function AlertsPage() {
|
||||||
title: 'Создан',
|
title: 'Создан',
|
||||||
dataIndex: 'created_at',
|
dataIndex: 'created_at',
|
||||||
key: 'created_at',
|
key: 'created_at',
|
||||||
|
width: 110,
|
||||||
render: (v: string) => (
|
render: (v: string) => (
|
||||||
<Tooltip title={dayjs(v).format('DD.MM.YYYY HH:mm:ss')}>
|
<Tooltip title={dayjs(v).format('DD.MM.YYYY HH:mm:ss')}>
|
||||||
{dayjs(v).format('DD.MM HH:mm')}
|
{dayjs(v).format('DD.MM HH:mm')}
|
||||||
|
|
@ -137,6 +158,7 @@ export function AlertsPage() {
|
||||||
{
|
{
|
||||||
title: 'Принят',
|
title: 'Принят',
|
||||||
key: 'acknowledged',
|
key: 'acknowledged',
|
||||||
|
width: 130,
|
||||||
render: (_: unknown, row: AlertItem) =>
|
render: (_: unknown, row: AlertItem) =>
|
||||||
row.acknowledged_at ? (
|
row.acknowledged_at ? (
|
||||||
<Space direction="vertical" size={0}>
|
<Space direction="vertical" size={0}>
|
||||||
|
|
@ -205,9 +227,7 @@ export function AlertsPage() {
|
||||||
<Typography.Title level={4} style={{ margin: 0 }}>
|
<Typography.Title level={4} style={{ margin: 0 }}>
|
||||||
Алерты
|
Алерты
|
||||||
</Typography.Title>
|
</Typography.Title>
|
||||||
{openCount > 0 && (
|
{openCount > 0 && <Badge count={openCount} color="red" />}
|
||||||
<Badge count={openCount} color="red" />
|
|
||||||
)}
|
|
||||||
</Space>
|
</Space>
|
||||||
</Col>
|
</Col>
|
||||||
<Col>
|
<Col>
|
||||||
|
|
@ -225,15 +245,60 @@ export function AlertsPage() {
|
||||||
</Col>
|
</Col>
|
||||||
</Row>
|
</Row>
|
||||||
|
|
||||||
<Table
|
{isLoading ? (
|
||||||
dataSource={alerts ?? []}
|
<Table loading columns={columns} dataSource={[]} rowKey="id" />
|
||||||
columns={columns}
|
) : (
|
||||||
rowKey="id"
|
groups.map((group) => {
|
||||||
loading={isLoading}
|
const openInGroup = group.alerts.filter((a) => a.status === 'open').length
|
||||||
size="middle"
|
return (
|
||||||
pagination={{ pageSize: 50 }}
|
<div key={group.object_id ?? 'unknown'} style={{ marginBottom: 24 }}>
|
||||||
rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')}
|
<Row justify="space-between" align="middle" style={{ marginBottom: 8 }}>
|
||||||
/>
|
<Col>
|
||||||
|
<Space align="center">
|
||||||
|
{group.object_id ? (
|
||||||
|
<Typography.Link
|
||||||
|
strong
|
||||||
|
onClick={() => navigate(`/work/objects/${group.object_id}`)}
|
||||||
|
>
|
||||||
|
{group.object_name ?? `Объект #${group.object_id}`}
|
||||||
|
</Typography.Link>
|
||||||
|
) : (
|
||||||
|
<Typography.Text strong>Без объекта</Typography.Text>
|
||||||
|
)}
|
||||||
|
<Badge count={group.alerts.length} color="blue" />
|
||||||
|
</Space>
|
||||||
|
</Col>
|
||||||
|
{openInGroup > 0 && group.object_id != null && (
|
||||||
|
<Col>
|
||||||
|
<Popconfirm
|
||||||
|
title={`Принять все ${openInGroup} открытых алертов объекта?`}
|
||||||
|
okText="Принять все"
|
||||||
|
cancelText="Отмена"
|
||||||
|
onConfirm={() => handleAcknowledgeByObject(group.object_id!)}
|
||||||
|
>
|
||||||
|
<Button
|
||||||
|
size="small"
|
||||||
|
icon={<ClockCircleOutlined />}
|
||||||
|
loading={acknowledgeByObject.isPending}
|
||||||
|
>
|
||||||
|
Принять все ({openInGroup})
|
||||||
|
</Button>
|
||||||
|
</Popconfirm>
|
||||||
|
</Col>
|
||||||
|
)}
|
||||||
|
</Row>
|
||||||
|
<Table
|
||||||
|
dataSource={group.alerts}
|
||||||
|
columns={columns}
|
||||||
|
rowKey="id"
|
||||||
|
size="small"
|
||||||
|
pagination={false}
|
||||||
|
rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue