Compare commits

..

No commits in common. "8115324988ced8b9dfd5639f57426b8145e793d4" and "48b1b737f33cd02b6bc205cf9942c9bb2cb190dc" have entirely different histories.

10 changed files with 57 additions and 356 deletions

View file

@ -1,25 +0,0 @@
"""0019 task results composite index
Revision ID: 0019
Revises: 0018
Create Date: 2026-05-06
"""
from alembic import op
revision = "0019"
down_revision = "0018"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_index(
"ix_task_results_task_device",
"task_results",
["task_id", "device_id"],
)
def downgrade() -> None:
op.drop_index("ix_task_results_task_device", table_name="task_results")

View file

@ -91,42 +91,6 @@ async def list_alerts(
return [_to_read(a) for a in result.scalars().all()]
@router.post("/acknowledge-by-object", response_model=list[AlertRead])
async def acknowledge_by_object(
object_id: int = Query(...),
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user),
):
result = await db.execute(
select(Alert)
.join(Device, Alert.device_id == Device.id)
.where(Device.object_id == object_id, Alert.status == "open")
)
alerts = result.scalars().all()
now = datetime.now(timezone.utc)
for alert in alerts:
alert.status = "acknowledged"
alert.acknowledged_at = now
alert.acknowledged_by = current_user.id
await db.flush()
alert_ids = [a.id for a in alerts]
loaded = []
for aid in alert_ids:
loaded.append(await _load_alert(db, aid))
for alert in loaded:
await sse_manager.publish(
SSEEvent(
type="alert_updated",
task_id=MONITOR_CHANNEL,
data={"alert_id": alert.id, "status": "acknowledged", "device_id": alert.device_id},
)
)
return [_to_read(a) for a in loaded]
@router.post("/{alert_id}/acknowledge", response_model=AlertRead)
async def acknowledge_alert(
alert_id: int,

View file

@ -549,19 +549,6 @@ async def run_dashboard_monitor_cycle(
", ".join(ips[:10]) + ("" if len(ips) > 10 else ""),
)
# Загружаем предыдущие ping-статусы до запуска — нужны для retry-логики
pre_ping_status_map: dict[int, str] = {}
if check.check_key == "ping":
pre_state_rows = (
await db.execute(
select(DashboardDeviceLiveState).where(
DashboardDeviceLiveState.check_key == "ping",
DashboardDeviceLiveState.device_id.in_([d.id for d in due_devices]),
)
)
).scalars().all()
pre_ping_status_map = {row.device_id: row.status for row in pre_state_rows}
sem = asyncio.Semaphore(max(1, check.concurrency))
tasks = [
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
@ -602,78 +589,6 @@ async def run_dashboard_monitor_cycle(
)
)
# Retry для offline-устройств (только ping):
# срабатывает только если устройство было online (переход online→offline)
if check.check_key == "ping":
offline_devices = [
r[0] for r in results
if r[2] == "offline" and pre_ping_status_map.get(r[0].id) == "online"
]
for retry_num in range(1, 3):
if not offline_devices:
break
logger.info(
"Dashboard monitor [ping] retry %d/2: %d offline устройств, ждём 60 сек",
retry_num, len(offline_devices),
)
await sse_manager.publish(
SSEEvent(
type="dashboard_monitor_update",
task_id=DASHBOARD_MONITOR_CHANNEL,
data={
"check_key": check.check_key,
"processed": completed_count,
"due": len(due_devices),
"failed": failed_count,
"retry": retry_num,
"retry_pending": len(offline_devices),
"ts": datetime.now(timezone.utc).isoformat(),
},
)
)
await asyncio.sleep(60)
retry_tasks = [
asyncio.ensure_future(_run_check_on_device(sem, check, device, device.object))
for device in offline_devices
]
retry_results = []
for coro in asyncio.as_completed(retry_tasks):
r = await coro
retry_results.append(r)
logger.debug(
"Dashboard monitor [ping] retry %d %s: %s",
retry_num, r[0].ip, r[2],
)
# Обновляем results: заменяем результат устройства если оно восстановилось
retry_map = {r[0].id: r for r in retry_results}
results = [
retry_map.get(r[0].id, r) for r in results
]
offline_devices = [r[0] for r in retry_results if r[2] == "offline"]
recovered = len(retry_results) - len(offline_devices)
failed_count = sum(1 for r in results if not r[1])
logger.info(
"Dashboard monitor [ping] retry %d/2 done: восстановилось %d, всё ещё offline %d",
retry_num, recovered, len(offline_devices),
)
await sse_manager.publish(
SSEEvent(
type="dashboard_monitor_update",
task_id=DASHBOARD_MONITOR_CHANNEL,
data={
"check_key": check.check_key,
"processed": len(due_devices),
"due": len(due_devices),
"failed": failed_count,
"retry": retry_num,
"retry_recovered": recovered,
"ts": datetime.now(timezone.utc).isoformat(),
},
)
)
async with AsyncSessionLocal() as dbw:
async with dbw.begin():
# Загружаем предыдущие статусы и baseline только для ping

View file

@ -87,6 +87,7 @@ async def _resolve_scope(scope: dict) -> list[tuple[Device, Object]]:
query = query.where(Device.object_id == scope["object_id"])
elif scope_type == "filter":
# Cross-object filter scope (from WorkDevices global page)
from app.models.object import Object as Obj
if scope.get("category"):
cats = scope["category"]
if isinstance(cats, list):
@ -95,12 +96,10 @@ async def _resolve_scope(scope: dict) -> list[tuple[Device, Object]]:
query = query.where(Device.category == cats)
if scope.get("object_id"):
query = query.where(Device.object_id == scope["object_id"])
if scope.get("city") or scope.get("client"):
query = query.join(Object, Device.object_id == Object.id)
if scope.get("city"):
query = query.where(Object.city == scope["city"])
if scope.get("client"):
query = query.where(Object.client == scope["client"])
if scope.get("city"):
query = query.where(Obj.city == scope["city"])
if scope.get("client"):
query = query.where(Obj.client == scope["client"])
else:
return []
@ -118,14 +117,20 @@ async def _run_on_device(
sem: asyncio.Semaphore,
) -> bool | None:
async with sem:
# Session 1: cancellation check + create TaskResult
started_at = datetime.now(timezone.utc)
# Bail out after acquiring the semaphore so large target lists don't
# stampede the DB pool with cancellation checks.
async with AsyncSessionLocal() as db:
task_row = await db.get(Task, task_id)
if task_row and task_row.cancel_requested:
return None # None = skipped due to cancellation
emit = lambda msg, level="info": _emit(task_id, msg, level)
# Create task_result row
result_id: int | None = None
started_at = datetime.now(timezone.utc)
async with AsyncSessionLocal() as db:
async with db.begin():
task_row = await db.get(Task, task_id)
if task_row and task_row.cancel_requested:
return None # None = skipped due to cancellation
tr = TaskResult(
task_id=task_id,
device_id=device.id,
@ -136,8 +141,6 @@ async def _run_on_device(
await db.flush()
result_id = tr.id
emit = lambda msg, level="info": _emit(task_id, msg, level)
try:
action_result = await action.execute(device, obj, params, emit, db=None)
except Exception as exc:
@ -158,7 +161,6 @@ async def _run_on_device(
finished_at = datetime.now(timezone.utc)
duration_ms = int((finished_at - started_at).total_seconds() * 1000)
# Session 2: update TaskResult + device status (ping) in one transaction
async with AsyncSessionLocal() as db:
async with db.begin():
tr = await db.get(TaskResult, result_id)
@ -170,8 +172,11 @@ async def _run_on_device(
tr.data = {**(action_data or {}), "error": action_error} if action_error else (action_data or None)
tr.duration_ms = duration_ms
if action.name == "ping":
new_status = "online" if action_result_success else "offline"
# For ping action: update device status, last_seen, last_ping_ms
if action.name == "ping":
new_status = "online" if action_result_success else "offline"
async with AsyncSessionLocal() as db:
async with db.begin():
device_row = await db.get(Device, device.id)
if device_row:
device_row.status = new_status
@ -181,10 +186,6 @@ async def _run_on_device(
ping_ms = (action_data or {}).get("ping_ms")
if ping_ms is not None:
device_row.last_ping_ms = ping_ms
# For ping action: publish SSE device_status after DB commit
if action.name == "ping":
new_status = "online" if action_result_success else "offline"
await sse_manager.publish(SSEEvent(
type="device_status",
task_id="__monitor__",
@ -216,17 +217,6 @@ async def _run_on_device(
return action_result_success
async def _heartbeat_writer(task_id: str, interval: int = 30) -> None:
"""Write task.heartbeat_at every `interval` seconds while task is running."""
while True:
await asyncio.sleep(interval)
async with AsyncSessionLocal() as db:
async with db.begin():
task = await db.get(Task, task_id)
if task and task.status == "running":
task.heartbeat_at = datetime.now(timezone.utc)
async def _run_task(task_id: str) -> None:
await _emit(task_id, "Task started", "info")
@ -237,7 +227,6 @@ async def _run_task(task_id: str) -> None:
return
task.status = "running"
task.started_at = datetime.now(timezone.utc)
task.heartbeat_at = datetime.now(timezone.utc)
scope = dict(task.target_scope)
action_type = task.type
params = dict(task.params)
@ -286,20 +275,10 @@ async def _run_task(task_id: str) -> None:
await _emit(task_id, f"Running on {len(targets)} device(s)...", "info")
sem = asyncio.Semaphore(_TASK_DEVICE_CONCURRENCY)
# Start heartbeat writer
heartbeat_task = asyncio.create_task(_heartbeat_writer(task_id))
try:
results = await asyncio.gather(
*[_run_on_device(task_id, device, obj, action, params, sem) for device, obj in targets],
return_exceptions=True,
)
finally:
heartbeat_task.cancel()
try:
await heartbeat_task
except asyncio.CancelledError:
pass
results = await asyncio.gather(
*[_run_on_device(task_id, device, obj, action, params, sem) for device, obj in targets],
return_exceptions=True,
)
ok = sum(1 for r in results if r is True)
skipped = sum(1 for r in results if r is None)

View file

@ -6,7 +6,6 @@ Runs once at app startup:
from datetime import datetime, timedelta, timezone
import sqlalchemy as sa
from sqlalchemy import update
from app.models.task import Task
@ -21,15 +20,7 @@ async def recover_stale_tasks() -> int:
async with db.begin():
result = await db.execute(
update(Task)
.where(
Task.status == "running",
# Use heartbeat_at when available (more accurate for long tasks),
# fall back to started_at for tasks that never wrote a heartbeat.
sa.or_(
sa.and_(Task.heartbeat_at.isnot(None), Task.heartbeat_at < cutoff),
sa.and_(Task.heartbeat_at.is_(None), Task.started_at < cutoff),
),
)
.where(Task.status == "running", Task.started_at < cutoff)
.values(status="stale")
.returning(Task.id)
)

View file

@ -50,16 +50,3 @@ export const useResolveAlert = () => {
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
})
}
export const useAcknowledgeByObject = () => {
const qc = useQueryClient()
return useMutation({
mutationFn: (objectId: number) =>
api
.post<AlertItem[]>(`/api/v1/alerts/acknowledge-by-object`, null, {
params: { object_id: objectId },
})
.then((r) => r.data),
onSuccess: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
})
}

View file

@ -1,6 +1,5 @@
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
import { api } from './client'
import { useSSE } from '../hooks/useSSE'
export interface TaskEvent {
id: number
@ -79,50 +78,11 @@ export const useTask = (id: string | undefined) =>
queryKey: ['tasks', id],
queryFn: () => api.get<TaskDetail>(`/api/v1/tasks/${id}`).then((r) => r.data),
enabled: !!id,
})
// Subscribe to SSE for a running task — merges results in cache, invalidates on completion.
export const useTaskSSE = (id: string | undefined) => {
const qc = useQueryClient()
useSSE(id, {
onTaskResult: (data) => {
qc.setQueryData<TaskDetail>(['tasks', id], (prev) => {
if (!prev) return prev
const existing = prev.device_results.find((r) => r.device_id === data.device_id)
const updated: TaskResult = existing
? { ...existing, status: data.status, stdout: data.stdout, stderr: data.stderr, duration_ms: data.duration_ms }
: {
id: 0,
device_id: data.device_id,
device_ip: data.device_ip,
device_hostname: data.device_hostname,
object_id: null,
object_name: null,
rack_id: null,
rack_name: null,
status: data.status,
stdout: data.stdout,
stderr: data.stderr,
exit_code: null,
data: null,
duration_ms: data.duration_ms,
executed_at: null,
}
return {
...prev,
device_results: existing
? prev.device_results.map((r) => (r.device_id === data.device_id ? updated : r))
: [...prev.device_results, updated],
}
})
},
onTaskStatus: () => {
// Task finished — do a full refetch to get final status and complete results
qc.invalidateQueries({ queryKey: ['tasks', id] })
qc.invalidateQueries({ queryKey: ['tasks'] })
refetchInterval: (query) => {
const status = query.state.data?.status
return status === 'running' || status === 'pending' ? 2000 : false
},
})
}
export const useActions = () =>
useQuery({
@ -173,5 +133,4 @@ export const useDeviceTasks = (deviceId: number) =>
api
.get<Task[]>('/api/v1/tasks', { params: { device_id: deviceId, limit: 100 } })
.then((r) => r.data),
refetchInterval: 10_000,
})

View file

@ -39,9 +39,8 @@ export function useSSE(taskId: string | null | undefined, handlers: SSEHandlers)
// ignore parse errors
}
},
onerror(err) {
onerror() {
ctrl.abort()
throw err
},
})

View file

@ -18,14 +18,13 @@ import {
Typography,
message,
} from 'antd'
import { useState, useMemo } from 'react'
import { useState } from 'react'
import { useQueryClient } from '@tanstack/react-query'
import { useNavigate } from 'react-router-dom'
import {
useAlerts,
useAcknowledgeAlert,
useResolveAlert,
useAcknowledgeByObject,
type AlertItem,
} from '../api/alerts'
import { useMonitorSSE } from '../hooks/useMonitorSSE'
@ -49,12 +48,6 @@ function AlertStatusTag({ status }: { status: string }) {
)
}
interface ObjectGroup {
object_id: number | null
object_name: string | null
alerts: AlertItem[]
}
export function AlertsPage() {
const navigate = useNavigate()
const qc = useQueryClient()
@ -68,28 +61,13 @@ export function AlertsPage() {
const acknowledge = useAcknowledgeAlert()
const resolve = useResolveAlert()
const acknowledgeByObject = useAcknowledgeByObject()
// Live updates via monitor SSE
useMonitorSSE({
onAlertCreated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
onAlertUpdated: () => qc.invalidateQueries({ queryKey: ['alerts'] }),
})
const groups = useMemo<ObjectGroup[]>(() => {
if (!alerts) return []
const map = new Map<number | null, ObjectGroup>()
for (const a of alerts) {
const key = a.object_id ?? null
if (!map.has(key)) {
map.set(key, { object_id: key, object_name: a.object_name, alerts: [] })
}
map.get(key)!.alerts.push(a)
}
return Array.from(map.values()).sort((a, b) =>
(a.object_name ?? '').localeCompare(b.object_name ?? '', 'ru')
)
}, [alerts])
const handleAcknowledge = async (id: number) => {
try {
await acknowledge.mutateAsync(id)
@ -108,15 +86,6 @@ export function AlertsPage() {
}
}
const handleAcknowledgeByObject = async (objectId: number) => {
try {
await acknowledgeByObject.mutateAsync(objectId)
message.success('Все алерты объекта приняты')
} catch {
message.error('Ошибка')
}
}
const columns = [
{
title: 'Статус',
@ -139,6 +108,17 @@ export function AlertsPage() {
</Space>
),
},
{
title: 'Объект',
dataIndex: 'object_name',
key: 'object_name',
render: (v: string | null, row: AlertItem) =>
v ? (
<a onClick={() => navigate(`/objects/${row.object_id}`)}>{v}</a>
) : (
'—'
),
},
{
title: 'Сообщение',
dataIndex: 'message',
@ -148,7 +128,6 @@ export function AlertsPage() {
title: 'Создан',
dataIndex: 'created_at',
key: 'created_at',
width: 110,
render: (v: string) => (
<Tooltip title={dayjs(v).format('DD.MM.YYYY HH:mm:ss')}>
{dayjs(v).format('DD.MM HH:mm')}
@ -158,7 +137,6 @@ export function AlertsPage() {
{
title: 'Принят',
key: 'acknowledged',
width: 130,
render: (_: unknown, row: AlertItem) =>
row.acknowledged_at ? (
<Space direction="vertical" size={0}>
@ -227,7 +205,9 @@ export function AlertsPage() {
<Typography.Title level={4} style={{ margin: 0 }}>
Алерты
</Typography.Title>
{openCount > 0 && <Badge count={openCount} color="red" />}
{openCount > 0 && (
<Badge count={openCount} color="red" />
)}
</Space>
</Col>
<Col>
@ -245,60 +225,15 @@ export function AlertsPage() {
</Col>
</Row>
{isLoading ? (
<Table loading columns={columns} dataSource={[]} rowKey="id" />
) : (
groups.map((group) => {
const openInGroup = group.alerts.filter((a) => a.status === 'open').length
return (
<div key={group.object_id ?? 'unknown'} style={{ marginBottom: 24 }}>
<Row justify="space-between" align="middle" style={{ marginBottom: 8 }}>
<Col>
<Space align="center">
{group.object_id ? (
<Typography.Link
strong
onClick={() => navigate(`/work/objects/${group.object_id}`)}
>
{group.object_name ?? `Объект #${group.object_id}`}
</Typography.Link>
) : (
<Typography.Text strong>Без объекта</Typography.Text>
)}
<Badge count={group.alerts.length} color="blue" />
</Space>
</Col>
{openInGroup > 0 && group.object_id != null && (
<Col>
<Popconfirm
title={`Принять все ${openInGroup} открытых алертов объекта?`}
okText="Принять все"
cancelText="Отмена"
onConfirm={() => handleAcknowledgeByObject(group.object_id!)}
>
<Button
size="small"
icon={<ClockCircleOutlined />}
loading={acknowledgeByObject.isPending}
>
Принять все ({openInGroup})
</Button>
</Popconfirm>
</Col>
)}
</Row>
<Table
dataSource={group.alerts}
columns={columns}
rowKey="id"
size="small"
pagination={false}
rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')}
/>
</div>
)
})
)}
<Table
dataSource={alerts ?? []}
columns={columns}
rowKey="id"
loading={isLoading}
size="middle"
pagination={{ pageSize: 50 }}
rowClassName={(row) => (row.status === 'open' ? 'alert-row-open' : '')}
/>
</div>
)
}

View file

@ -45,7 +45,7 @@ import {
} from '../api/objects'
import { useRacks, useCreateRack, useUpdateRack, useDeleteRack, type RackItem } from '../api/racks'
import { useZones, useCreateZone, useDeleteZone, type ZoneItem } from '../api/zones'
import { useActions, useCreateTask, useTask, useTaskSSE } from '../api/tasks'
import { useActions, useCreateTask, useTask } from '../api/tasks'
import { CategoryTag } from '../components/StatusBadge'
import { DeviceEditModal } from '../components/DeviceEditModal'
import { DiscoveryDrawer } from '../components/DiscoveryDrawer'
@ -90,11 +90,8 @@ function PingButton({
const [taskId, setTaskId] = useState<string | null>(null)
const queryClient = useQueryClient()
const createTask = useCreateTask()
useTaskSSE(taskId ?? undefined)
// When task finishes (SSE invalidates ['tasks', taskId]), refresh devices and clear taskId
const { data: task } = useTask(taskId ?? undefined)
useEffect(() => {
if (task && ['success', 'failed', 'partial', 'stale', 'cancelled'].includes(task.status)) {
queryClient.invalidateQueries({ queryKey: ['devices', objId] })