From 8d9ebc0446cfd319720108f595b17e92e7ceb091 Mon Sep 17 00:00:00 2001 From: dv Date: Wed, 29 Apr 2026 12:31:35 +0300 Subject: [PATCH] =?UTF-8?q?+=D1=80=D0=BE=D0=B4=D0=B0=D1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 255 ++++++++++++++++++ .../app/services/discovery/config_parser.py | 29 ++ .../app/services/discovery/orchestrator.py | 4 +- 3 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..40815c4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,255 @@ +# UTP Service — Project Reference + +Fleet management web service for parking/access control equipment (CAPS system). + +## Stack + +| Layer | Tech | +|---|---| +| Backend | FastAPI + SQLAlchemy 2.0 async (asyncpg), Alembic, asyncssh, psycopg3 | +| Frontend | React 18 + TypeScript, Vite, Ant Design 5, TanStack React Query 5, Zustand | +| DB | PostgreSQL 16 | +| Infra | Docker Compose | + +Run: `docker compose up` — backend :8005, frontend :3005, postgres :5432 + +## Project Layout + +``` +utp_service/ +├── backend/ +│ ├── app/ +│ │ ├── main.py # FastAPI app, routers, lifespan +│ │ ├── config.py # Settings (DATABASE_URL, SECRET_KEY, ENCRYPTION_KEY) +│ │ ├── dependencies.py # get_db, get_current_user +│ │ ├── models/ # SQLAlchemy ORM models +│ │ ├── routers/ # API route handlers +│ │ ├── schemas/ # Pydantic request/response schemas +│ │ ├── services/ # Business logic +│ │ ├── plugins/ # Plugin registry + builtin plugins +│ │ ├── workers/ # Background workers (monitor, startup) +│ │ └── middleware/auth.py # Auth middleware +│ └── alembic/versions/ # Numbered migrations 0001–0009 +└── frontend/ + └── src/ + ├── App.tsx # React Router routes + ├── pages/ # Page-level components + ├── api/ # React Query hooks + API calls + ├── hooks/ # useSSE, useMonitorSSE + ├── components/ # AppLayout, StatusBadge, DeviceEditModal + └── store/auth.ts # Zustand auth store +``` + +## Database Session Pattern + +`autoflush=False` — must call `await db.flush()` explicitly after `db.add()` before any SELECT that needs to see the new rows. `session.begin()` auto-commits on handler return. + +```python +async def get_db() -> AsyncGenerator[AsyncSession, None]: + async with AsyncSessionLocal() as session: + async with session.begin(): + yield session +``` + +## Models + +### Object (`objects` table) +Represents a physical site/parking lot. + +Key fields: `name`, `description`, `city`, `client`, `server_ip` (main CAPS server IP), +`db_host/port/name/user/db_pass_enc/db_table/db_via_tunnel` (CAPS PostgreSQL connection), +`ssh_user/ssh_pass_enc/ssh_port` (SSH credentials, shared across site), +`is_active`, `tags` (M2M → Tag) + +### Device (`devices` table) +Network device at a site. + +Key fields: `object_id` (FK), `ip` (unique per object: `uq_device_object_ip`), +`hostname`, `mac`, `category`, `role`, `location`, `vendor`, `model`, +`source` (`manual`/`db_sync`/`auto_ssh`/`csv`/`sheets`), +`status` (`unknown`/`online`/`offline`/`degraded`), `monitor_enabled`, +`rack_id` (FK → Rack), `connection_params` (JSONB), +`device_meta` (JSONB — e.g. `{"plugin": "camera_1", "cls": "..."}`), +`capabilities` (ARRAY), `ssh_user_override/ssh_pass_override_enc/ssh_port_override` + +Device categories: `main_server`, `vm`, `router`, `embedded`, `camera`, `io_board`, `bank_terminal`, `cash_register`, `other` +Device roles: `plate`, `face`, `overview`, `other` +Device locations: `server_room`, `rack`, `street` + +### Rack (`racks` table) +`id`, `object_id`, `name`, `description`, `location`, `zone_id` (FK → Zone) + +### Zone (`zones` table) +`id`, `object_id`, `name`, `description`, racks relationship + +### Task (`tasks` table) +`id`, `type`, `status` (pending/running/success/partial/failed/stale/cancelled), +`target_scope` (JSONB — see Task System), `params` (JSONB), `result` (JSONB), +`created_by`, timestamps, → TaskEvent, TaskResult relationships + +### Alert (`alerts` table) +`id`, `device_id`, `status` (open/acknowledged/resolved), `message`, timestamps + +### ConfigSnapshot / SnapshotContent +Content-addressed: `SnapshotContent(sha256 PK, content)`, `ConfigSnapshot(device_id, sha256 FK, path, captured_at)` + +### Tag (`tags` table) +`id`, `name`, `color`, `tag_type`; M2M with objects via `object_tags` + +## Routers (all under `/api/v1/`) + +| Router file | Prefix | Key endpoints | +|---|---|---| +| auth.py | /auth | POST login, POST refresh, POST logout | +| objects.py | /objects | CRUD + GET /meta (cities/clients) + POST /{id}/discover + POST /{id}/sync | +| devices.py | /objects/{id}/devices | CRUD + POST /import + POST /import/preview | +| global_devices.py | /devices | GET all devices across objects (filters: category, city, client, object_id, status) | +| racks.py | /objects/{id}/racks | CRUD | +| zones.py | /objects/{id}/zones | CRUD | +| tasks.py | /tasks | POST create + GET list/detail + GET /{id}/events (SSE); GET list accepts `?device_id=X` (JOIN task_results) | +| events.py | /events | GET SSE stream (task_id param) | +| alerts.py | /alerts | GET list (filters: status, object_id, device_id) + POST /{id}/acknowledge + POST /{id}/resolve | +| snapshots.py | /objects/{id}/snapshots | GET list (filter: device_id) + GET /diff?a=&b= + GET /{id} | +| tags.py | /tags | CRUD | +| admin.py | /admin | User mgmt + plugin control + health | + +## Services + +| File | Purpose | +|---|---| +| device_discovery.py | SSH-based discovery: SSH → rack PC → parse /etc/caps/config.ini → extract plugin IPs → upsert devices | +| object_db.py | Sync devices from CAPS PostgreSQL DB | +| task.py | Task dispatch, per-device async execution (concurrency: 20) | +| ssh.py | asyncssh client with concurrency limits | +| crypto.py | Fernet encrypt/decrypt for passwords (db_pass_enc, ssh_pass_enc) | +| sse.py | SSEManager pub/sub for Server-Sent Events | +| camera.py | Hikvision ISAPI HTTP client | +| snapshot.py | SHA-256 content-addressed config snapshot storage | +| sheets.py | Google Sheets device import (gspread) | +| auth.py | JWT + bcrypt, access (15min) + refresh (7d) tokens | + +## SSH Discovery (`device_discovery.py`) + +Flow: +1. Resolve CAPS DB connection: use explicit `obj.db_*` fields OR autodetect by SSHing into `obj.server_ip` → read `/etc/caps/config.ini` → extract `db_uri` +2. Query racks table → get `(external_id, rack_host_ip, rack_name)` tuples +3. For each rack: SSH into `rack_host_ip` → read `/etc/caps/config.ini` +4. Parse config with `RawConfigParser(strict=False)` (strict=False needed for duplicate keys like `ini=` in uwsgi) +5. Extract devices from `[plugin:*]` sections by parsing URI fields (tcp://, http://) — skip serial://, localhost +6. Upsert: SELECT by `(object_id, ip)`, INSERT or UPDATE category/role/hostname +7. `await db.flush()` after each rack (autoflush=False — without flush, SELECT won't see pending db.add() rows) + +Hostname format: `rack{rack_name}-{plugin_name}` e.g. `rack01-camera_grz` + +## Task System + +`target_scope` JSONB shapes: +- `{"type": "device", "id": 123}` +- `{"type": "object", "id": 4}` +- `{"type": "category", "object_id": 4, "category": "main_server"}` +- `{"type": "device_list", "ids": [1, 2, 3]}` + +Task events streamed via SSE at `GET /api/v1/events?task_id={id}` +Monitor events at `GET /api/v1/events?task_id=__monitor__` + +SSE fix: `onerror` must `throw err` to prevent fetchEventSource auto-retry loop on 401. + +## Frontend Routes + +``` +/ → redirect to /inventory/objects +/inventory/objects → ObjectsPage (mode=inventory) — city-grouped list +/inventory/objects/:id → InventoryObjectDetailPage — device CRUD, SSH discovery +/inventory/objects/:id/devices/:deviceId → DeviceDetailPage — device card (4 tabs) +/inventory/objects/:id/snapshots → SnapshotsPage +/work/objects → ObjectsPage (mode=work) — city-grouped list +/work/objects/:id → WorkObjectDetailPage — task execution, device status +/work/devices → WorkDevicesPage — global device list with filters +``` + +Layout nav: ИНВЕНТАРЬ (DatabaseOutlined) | РАБОТА (ControlOutlined, AppstoreOutlined) + +### DeviceDetailPage (`pages/DeviceDetail.tsx`) + +URL: `/inventory/objects/:id/devices/:deviceId` + +Шапка: IP + live-статус (SSE via `useMonitorSSE`), hostname, теги категории/стойки/вендора, кнопки Редактировать / Удалить. + +Вкладки: +- **Основное** — все поля `DeviceItem` через `Descriptions`, `device_meta`/`connection_params` в JSON-блоках, capabilities тегами +- **Алерты** — список алертов устройства (`useAlerts({ device_id })`), фильтр по статусу, кнопки acknowledge/resolve +- **Снапшоты** — список (`useSnapshots(objId, { device_id })`), выбор двух → цветной diff-viewer через `useSnapshotDiff` +- **История задач** — задачи через `useDeviceTasks(deviceId)`, раскрытие строки показывает stdout/stderr из `TaskResult` + +### DeviceEditModal (`components/DeviceEditModal.tsx`) + +Самодостаточный компонент формы редактирования/добавления устройства. Используется в: +- `InventoryObjectDetailPage` (кнопки Edit и «Добавить устройство») +- `DeviceDetailPage` (кнопка «Редактировать» в шапке) + +Props: `open`, `device: DeviceItem | null` (null = режим добавления), `objId`, `onClose`, `onSaved?` + +Внутри: `useCreateDevice` / `useUpdateDevice`, `useRacks` для списка стоек. + +## Frontend API Hooks (`src/api/`) + +- `objects.ts`: useObjects, useObject, useCreateObject, useUpdateObject, useDeleteObject, useObjectsMeta, + useDiscoverObject, useSyncObject, useSyncSheets, usePreviewCSV, useImportCSV, + **useDevices(objId)** — список устройств объекта, + **useDevice(objId, deviceId)** — одно устройство (queryKey: `['device', objId, deviceId]`), + useCreateDevice, useUpdateDevice (инвалидирует `['devices', objId]` + `['device', objId, id]`), useDeleteDevice, + useDeleteAllDevices, useAllDevices (global, фильтры: category/city/client/object_id/status) +- `tasks.ts`: useTasks, useTask, useActions, useCreateTask, + **useDeviceTasks(deviceId)** — задачи по устройству (queryKey: `['tasks', 'device', deviceId]`) +- `alerts.ts`: useAlerts (params: status, object_id, **device_id**, limit, skip), useAcknowledgeAlert, useResolveAlert +- `snapshots.ts`: useSnapshots(objId, params?: {**device_id**?, label?, limit?, skip?}), useSnapshotDetail, useSnapshotDiff(objId, a, b) +- `racks.ts`: useRacks, useCreateRack, useUpdateRack, useDeleteRack +- `zones.ts`: useZones, useCreateZone, useDeleteZone + +### DeviceItem interface (полный, `api/objects.ts`) + +```typescript +interface DeviceItem { + id, object_id, ip, hostname, mac + vendor, model, firmware_version, external_id + category, role, source + rack_id, rack_name, rack_type, location + ssh_user_override, ssh_port_override + device_meta: Record // JSONB + connection_params: Record // JSONB + capabilities: string[] + status, last_seen, last_ping_ms, monitor_enabled + is_active, notes, created_at, updated_at +} +``` + +## Plugins (builtin) + +Located in `backend/app/plugins/builtin/`: +- `camera/`: get_status (Hikvision ISAPI), set_ntp +- `mikrotik/`: get_config, get_ntp, set_ntp (RouterOS via SSH) +- `ssh/`: debian_system_info, get_file, get_timedatectl, ssh_command +- `common/`: ping + +## Migrations + +Apply via: `docker compose exec backend alembic upgrade head` + +Files in `backend/alembic/versions/`: +- 0001: initial schema +- 0002: tasks +- 0003: racks +- 0004: zones +- 0005: snapshots +- 0006: alerts +- 0007: plugins +- 0008: inventory (vendor, location, connection_params on devices) +- 0009: city, client on objects + +## Known Gotchas + +- `autoflush=False`: always `await db.flush()` after `db.add()` before subsequent SELECTs in the same transaction +- `await db.refresh(obj)` — no attribute_names arg, otherwise server-generated columns (updated_at) aren't refreshed +- `RawConfigParser(strict=False)` — CAPS configs have duplicate keys in [uwsgi] section +- SSH discovery: `onerror(err) { throw err }` in fetchEventSource to stop auto-retry on non-200 +- Encrypted fields stored as Fernet ciphertext; use `crypto.encrypt()` / `crypto.decrypt()` diff --git a/backend/app/services/discovery/config_parser.py b/backend/app/services/discovery/config_parser.py index 9b0d28d..987c1d6 100644 --- a/backend/app/services/discovery/config_parser.py +++ b/backend/app/services/discovery/config_parser.py @@ -25,6 +25,7 @@ def parse_config( config_text: str, rack_host: str, all_rack_hosts: list[str] | None = None, + main_server_ip: str | None = None, ) -> list[DiscoveredDevice]: """Extract discovered network devices from a CAPS config file. @@ -32,6 +33,7 @@ def parse_config( config_text: Config file content. rack_host: IP of the current rack PC (excluded from results). all_rack_hosts: All rack IPs (excluded so rack servers aren't mistaken for devices). + main_server_ip: Main CAPS server IP (excluded for service plugins that point back to it). """ if all_rack_hosts is None: all_rack_hosts = [rack_host] @@ -59,6 +61,33 @@ def parse_config( continue plugin_name = section[len("plugin:"):] + + if plugin_name.startswith("recognition"): + raw = parser.get(section, "uri", fallback=None) + if raw: + logger.info("[discovery] Plugin '%s' recognition uri='%s'", plugin_name, raw) + info = extract_device_info(raw) + if info: + ip, http_user, http_pass = info + if ip in all_rack_hosts: + logger.info("[discovery] -> IGNORED (is a rack server IP: %s)", ip) + elif main_server_ip and ip == main_server_ip: + logger.info("[discovery] -> IGNORED (is main server IP: %s)", ip) + elif ip in seen_ips: + logger.info("[discovery] -> IGNORED (duplicate)") + else: + seen_ips.add(ip) + logger.info("[discovery] -> EXTRACTED %s as RoadAR (category=vm)", ip) + discovered.append(DiscoveredDevice( + ip=ip, + category="vm", + role="other", + plugin_name="roadar", + cls=parser.get(section, "cls", fallback="recognition"), + http_username=http_user, + http_password=http_pass, + )) + if any(plugin_name.startswith(prefix) for prefix in _SKIP_PLUGIN_PREFIXES): logger.info("[discovery] Plugin '%s' - IGNORED (skip list)", plugin_name) continue diff --git a/backend/app/services/discovery/orchestrator.py b/backend/app/services/discovery/orchestrator.py index 4b286b7..908b5f1 100644 --- a/backend/app/services/discovery/orchestrator.py +++ b/backend/app/services/discovery/orchestrator.py @@ -157,7 +157,7 @@ async def discover_via_ssh( discovered_device_ips.add(rack_host) await _emit("disc_action", {"message": f"Парсинг конфигурации {rack_name}..."}) - devices_found = parse_config(config_text, rack_host, all_rack_ips) + devices_found = parse_config(config_text, rack_host, all_rack_ips, obj.server_ip) logger.info("[discovery] Found %d devices from plugins in %s", len(devices_found), rack_host) _rack_found = len(devices_found) @@ -725,7 +725,7 @@ async def _discover_slave_racks( await _upsert_slave_device(db, obj, candidate_ip, slave_identifier, slave_rack_id, master_ip_for_slave, now, result) all_rack_ips_with_slave = list(all_rack_ips_set) + [candidate_ip] - slave_devices = parse_config(slave_config, candidate_ip, all_rack_ips_with_slave) + slave_devices = parse_config(slave_config, candidate_ip, all_rack_ips_with_slave, obj.server_ip) logger.info("[discovery] Slave %s: found %d plugin devices", candidate_ip, len(slave_devices)) await _emit("disc_slave_found", { "rack_name": slave_identifier, "ip": candidate_ip, "found": len(slave_devices)