927 lines
39 KiB
Python
927 lines
39 KiB
Python
"""
|
|
Main discovery orchestrator.
|
|
|
|
Coordinates all phases of SSH-based device discovery:
|
|
0. Resolve DB connection (explicit config or autodetect via SSH).
|
|
0.5 Pre-upsert the DB server device (so users can configure tunnels on failure).
|
|
1. Fetch rack hosts from the CAPS DB; sync Rack table entries.
|
|
1b. Upsert infrastructure devices (main_server).
|
|
2. SSH into each rack → parse CAPS config → upsert plugin devices.
|
|
2.5 Discover slave racks via ARP neighbor tables.
|
|
3. Discover MikroTik via SSH_CLIENT env var.
|
|
4. Discover Proxmox via port 8006 scan.
|
|
"""
|
|
|
|
import logging
|
|
from configparser import RawConfigParser
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.device import Device
|
|
from app.models.object import Object
|
|
from app.models.rack import Rack
|
|
from app.services.crypto import decrypt, encrypt
|
|
|
|
from .config_parser import parse_config, parse_slave_check
|
|
from .db_resolver import autodetect_db_from_server, get_rack_hosts
|
|
from .ssh_client import (
|
|
discover_mikrotik_multi,
|
|
discover_proxmox_via_port_scan,
|
|
fetch_slave_config,
|
|
get_ip_neighbors_multi,
|
|
ssh_run_multi,
|
|
)
|
|
from .types import DeviceAction, DiscoveredDevice, DiscoveryResult, ProgressCallback
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ── Public entry point ────────────────────────────────────────────────────────
|
|
|
|
async def discover_via_ssh(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
config_path: str = "/etc/caps/config.ini",
|
|
progress_cb: ProgressCallback | None = None,
|
|
) -> DiscoveryResult:
|
|
result = DiscoveryResult()
|
|
|
|
async def _emit(event_type: str, data: dict) -> None:
|
|
if progress_cb is not None:
|
|
await progress_cb(event_type, data)
|
|
|
|
from app.services.ssh import build_object_auth_options
|
|
ssh_options = await build_object_auth_options(obj, db)
|
|
|
|
if not ssh_options:
|
|
result.errors.append("SSH credentials not configured on this object")
|
|
return result
|
|
|
|
ssh_port = obj.ssh_port or 22
|
|
# First option's password is used by single-auth Proxmox scan (legacy).
|
|
_first = ssh_options[0]
|
|
ssh_password = _first.password if _first.type == "password" else ""
|
|
|
|
# ── Phase 0: Resolve DB connection ────────────────────────────────────────
|
|
db_params = await _resolve_db_connection(obj, ssh_port, ssh_options, result, _emit)
|
|
if db_params is None:
|
|
return result
|
|
|
|
db_host, db_port, db_name, db_user, db_password, db_table, server_ip_ssh_verified = db_params
|
|
|
|
# ── Phase 0.5: Pre-upsert DB server device ────────────────────────────────
|
|
now = datetime.now(timezone.utc)
|
|
db_device_obj, use_tunnel = await _pre_upsert_db_device(
|
|
db, obj, db_host, result, now
|
|
)
|
|
|
|
# ── Phase 1: Fetch rack hosts ──────────────────────────────────────────────
|
|
await _emit("disc_action", {"message": "Получение списка стоек из БД..."})
|
|
logger.info(
|
|
"[discovery] Connecting to object DB: host=%s port=%s dbname=%s user=%s table=%s via_tunnel=%s",
|
|
db_host, db_port, db_name, db_user, db_table, use_tunnel,
|
|
)
|
|
try:
|
|
rack_hosts = await get_rack_hosts(
|
|
host=db_host, port=db_port, user=db_user,
|
|
password=db_password or "", dbname=db_name, table=db_table,
|
|
via_tunnel=use_tunnel,
|
|
tunnel_host=obj.server_ip, tunnel_ssh_port=ssh_port,
|
|
ssh_options=ssh_options,
|
|
)
|
|
if db_device_obj is not None:
|
|
db_device_obj.status = "online"
|
|
db_device_obj.last_seen = now
|
|
except Exception as exc:
|
|
if db_device_obj is not None:
|
|
db_device_obj.status = "offline"
|
|
err_msg = f"Cannot fetch rack hosts from object DB: {exc}"
|
|
if "timeout" in str(exc).lower():
|
|
err_msg += ". Попробуйте настроить SSH-туннель (отредактируйте устройство БД и включите опцию)"
|
|
result.errors.append(err_msg)
|
|
return result
|
|
|
|
if not rack_hosts:
|
|
result.errors.append("No racks with host IPs found in object DB")
|
|
return result
|
|
|
|
await _emit("disc_server", {"rack_count": len(rack_hosts)})
|
|
|
|
# ── Phase 1a: Sync Rack table entries ──────────────────────────────────────
|
|
rack_id_mapping = await _sync_rack_entries(db, obj, rack_hosts)
|
|
|
|
# ── Phase 1b: Upsert infrastructure devices ───────────────────────────────
|
|
infra_devices = []
|
|
if obj.server_ip:
|
|
infra_devices.append((obj.server_ip, "main_server", "caps_server", server_ip_ssh_verified))
|
|
await _upsert_infra_devices(db, obj, infra_devices, result, now)
|
|
|
|
# ── Phase 2: Process racks ────────────────────────────────────────────────
|
|
neighbor_candidates: set[str] = set()
|
|
discovered_device_ips: set[str] = set()
|
|
|
|
for rack_idx, (external_id, rack_host, rack_name, _rack_type_db) in enumerate(rack_hosts):
|
|
await _emit("disc_rack_start", {
|
|
"rack_name": rack_name, "rack_host": rack_host,
|
|
"index": rack_idx, "total": len(rack_hosts),
|
|
})
|
|
|
|
# Allow per-device config_path override stored in connection_params.
|
|
effective_config_path = await _resolve_rack_config_path(
|
|
db, obj, rack_host, config_path
|
|
)
|
|
|
|
rack_ssh_ok, config_text = await _ssh_fetch_rack_config(
|
|
rack_host, ssh_port, ssh_options, effective_config_path, rack_name, result, _emit
|
|
)
|
|
|
|
await _upsert_rack_device(
|
|
db, obj, external_id, rack_host, rack_name,
|
|
rack_id_mapping, rack_ssh_ok, now, result
|
|
)
|
|
|
|
if not rack_ssh_ok:
|
|
try:
|
|
await db.flush()
|
|
except Exception as exc:
|
|
result.errors.append(f"Flush error for offline rack {rack_host}: {exc}")
|
|
await _emit("disc_rack_done", {
|
|
"rack_name": rack_name, "rack_host": rack_host, "found": 0, "ssh_ok": False
|
|
})
|
|
continue
|
|
|
|
neigh_ips = await get_ip_neighbors_multi(rack_host, ssh_port, ssh_options)
|
|
neighbor_candidates.update(neigh_ips)
|
|
discovered_device_ips.add(rack_host)
|
|
|
|
all_rack_ips = [h for _, h, _, _ in rack_hosts]
|
|
await _emit("disc_action", {"message": f"Парсинг конфигурации {rack_name}..."})
|
|
devices_found = parse_config(config_text, rack_host, all_rack_ips)
|
|
logger.info("[discovery] Found %d devices from plugins in %s", len(devices_found), rack_host)
|
|
|
|
for found in devices_found:
|
|
await _upsert_plugin_device(
|
|
db, obj, found, external_id, rack_id_mapping, result
|
|
)
|
|
discovered_device_ips.add(found.ip)
|
|
|
|
try:
|
|
await db.flush()
|
|
except Exception as exc:
|
|
result.errors.append(f"Flush error after rack {rack_host}: {exc}")
|
|
await _emit("disc_rack_done", {
|
|
"rack_name": rack_name, "rack_host": rack_host,
|
|
"found": len(devices_found), "ssh_ok": True,
|
|
})
|
|
|
|
# ── Phase 2.5: Slave rack discovery via ARP ───────────────────────────────
|
|
all_rack_ips_set = {h for _, h, _, _ in rack_hosts}
|
|
infra_ips = {ip for ip in [obj.server_ip, db_host] if ip}
|
|
common_gateways = {ip for ip in neighbor_candidates if ip.endswith(".1") or ip.endswith(".254")}
|
|
slave_candidates = (
|
|
neighbor_candidates - all_rack_ips_set - discovered_device_ips - infra_ips - common_gateways
|
|
)
|
|
|
|
await _emit("disc_phase", {"phase": "slaves", "candidate_count": len(slave_candidates)})
|
|
if slave_candidates:
|
|
logger.info("[discovery] Phase 2.5: Slave discovery — %d candidate(s)", len(slave_candidates))
|
|
await _emit("disc_action", {
|
|
"message": f"Поиск слейв-стоек: проверяем {len(slave_candidates)} ARP-кандидатов..."
|
|
})
|
|
else:
|
|
await _emit("disc_action", {"message": "Слейв-стойки: ARP-кандидатов не найдено"})
|
|
|
|
await _discover_slave_racks(
|
|
db, obj, slave_candidates, rack_hosts, ssh_port, ssh_options,
|
|
config_path, all_rack_ips_set, now, result, _emit
|
|
)
|
|
|
|
# ── Phase 3: MikroTik via SSH_CLIENT ──────────────────────────────────────
|
|
await _emit("disc_phase", {"phase": "mikrotik"})
|
|
await _emit("disc_action", {"message": "Поиск MikroTik (SSH_CLIENT)..."})
|
|
await _discover_mikrotik(db, obj, rack_hosts, ssh_port, ssh_options, result, _emit)
|
|
|
|
# ── Phase 4: Proxmox via port 8006 scan ───────────────────────────────────
|
|
await _emit("disc_phase", {"phase": "proxmox"})
|
|
await _emit("disc_action", {"message": "Сканирование порта 8006 (Proxmox)..."})
|
|
await _discover_proxmox(db, obj, ssh_port, ssh_password, rack_hosts, result, _emit)
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────
|
|
_log_summary(result)
|
|
return result
|
|
|
|
|
|
# ── Phase helpers ─────────────────────────────────────────────────────────────
|
|
|
|
async def _resolve_db_connection(
|
|
obj: Object,
|
|
ssh_port: int,
|
|
ssh_options: list,
|
|
result: DiscoveryResult,
|
|
_emit,
|
|
) -> tuple | None:
|
|
"""Determine the DB connection parameters.
|
|
|
|
Returns (db_host, db_port, db_name, db_user, db_password, db_table, server_ip_ssh_verified)
|
|
or None if resolution failed (errors added to result).
|
|
"""
|
|
db_host = obj.db_host
|
|
db_port = obj.db_port or 5432
|
|
db_name = obj.db_name
|
|
db_user = obj.db_user
|
|
db_password: str | None = None
|
|
db_table = obj.db_table
|
|
server_ip_ssh_verified = False
|
|
|
|
if db_host and db_name and db_user and obj.db_pass_enc and db_table:
|
|
try:
|
|
db_password = decrypt(obj.db_pass_enc)
|
|
except Exception:
|
|
result.errors.append("Failed to decrypt DB password")
|
|
return None
|
|
elif obj.server_ip:
|
|
await _emit("disc_action", {"message": f"Чтение конфигурации сервера {obj.server_ip}..."})
|
|
detected = await autodetect_db_from_server(
|
|
server_ip=obj.server_ip, ssh_port=ssh_port, ssh_options=ssh_options,
|
|
)
|
|
if not detected or not all([detected.get("host"), detected.get("user"), detected.get("dbname")]):
|
|
await _emit_db_autodetect_error(obj, ssh_port, ssh_options, result)
|
|
return None
|
|
|
|
db_host = detected["host"]
|
|
db_port = detected["port"]
|
|
db_user = detected["user"]
|
|
db_password = detected["password"]
|
|
db_name = detected["dbname"]
|
|
db_table = db_table or obj.db_table or "racks"
|
|
server_ip_ssh_verified = True
|
|
else:
|
|
result.errors.append(
|
|
"DB connection not configured. Either fill in the DB fields on the object "
|
|
"or set server_ip so the DB can be auto-detected via SSH."
|
|
)
|
|
return None
|
|
|
|
return db_host, db_port, db_name, db_user, db_password, db_table, server_ip_ssh_verified
|
|
|
|
|
|
async def _emit_db_autodetect_error(obj, ssh_port, ssh_options, result: DiscoveryResult) -> None:
|
|
try:
|
|
raw = await ssh_run_multi(obj.server_ip, ssh_port, ssh_options, "cat /etc/caps/config.ini")
|
|
diag_parser = RawConfigParser(strict=False)
|
|
diag_parser.read_string(raw)
|
|
found_sections = diag_parser.sections()
|
|
found_keys = {s: list(diag_parser.options(s)) for s in found_sections}
|
|
result.errors.append(
|
|
f"db_uri not found in server config. "
|
|
f"Sections found: {found_sections}. "
|
|
f"Keys per section: {found_keys}. "
|
|
f"Fill in DB fields manually on the object."
|
|
)
|
|
except Exception as diag_exc:
|
|
result.errors.append(
|
|
f"Could not read server config for auto-detection: {diag_exc}. "
|
|
f"Fill in DB fields manually on the object."
|
|
)
|
|
|
|
|
|
async def _pre_upsert_db_device(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
db_host: str | None,
|
|
result: DiscoveryResult,
|
|
now: datetime,
|
|
) -> tuple:
|
|
"""Upsert the CAPS DB server as a device before the DB connection attempt.
|
|
|
|
Returns (db_device_obj, use_tunnel).
|
|
"""
|
|
db_device_obj: Device | None = None
|
|
if db_host and db_host != obj.server_ip:
|
|
try:
|
|
db_device_obj = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == db_host)
|
|
)).scalar_one_or_none()
|
|
if db_device_obj is None:
|
|
db_device_obj = Device(
|
|
object_id=obj.id, ip=db_host, hostname="db_server",
|
|
category="vm", role="other", source="auto_ssh",
|
|
location="server_room", status="unknown",
|
|
device_meta={"is_caps_db": True},
|
|
)
|
|
db.add(db_device_obj)
|
|
await db.flush()
|
|
logger.info("[discovery] %s (db_server) - PRE-ADDED before connection attempt", db_host)
|
|
result.created += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=db_host, hostname="db_server", action="ADDED",
|
|
reason="object infrastructure device (vm/db_server)",
|
|
category="vm", role="other",
|
|
))
|
|
except Exception as exc:
|
|
logger.warning("[discovery] Failed to pre-upsert db_server %s: %s", db_host, exc)
|
|
|
|
device_via_tunnel = bool(
|
|
(db_device_obj.connection_params or {}).get("db_via_tunnel")
|
|
) if db_device_obj else False
|
|
use_tunnel = device_via_tunnel or obj.db_via_tunnel
|
|
|
|
return db_device_obj, use_tunnel
|
|
|
|
|
|
async def _sync_rack_entries(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
rack_hosts: list[tuple],
|
|
) -> dict[str, int]:
|
|
"""Ensure a Rack row exists for every rack returned from the CAPS DB.
|
|
|
|
Returns a mapping of external_id → Rack.id.
|
|
"""
|
|
rack_id_mapping: dict[str, int] = {}
|
|
for ext_id, _, rack_name, rack_type_db in rack_hosts:
|
|
existing = (await db.execute(
|
|
select(Rack).where(
|
|
Rack.object_id == obj.id,
|
|
Rack.name.in_([ext_id, rack_name]),
|
|
)
|
|
)).scalar_one_or_none()
|
|
if existing:
|
|
rack_id_mapping[ext_id] = existing.id
|
|
if existing.name != rack_name:
|
|
logger.info("[discovery] Renamed rack '%s' → '%s'", existing.name, rack_name)
|
|
existing.name = rack_name
|
|
if rack_type_db and not existing.rack_type:
|
|
existing.rack_type = rack_type_db
|
|
else:
|
|
new_rack = Rack(object_id=obj.id, name=rack_name, rack_type=rack_type_db)
|
|
db.add(new_rack)
|
|
await db.flush()
|
|
rack_id_mapping[ext_id] = new_rack.id
|
|
logger.info("[discovery] Created rack '%s' rack_type=%s (id=%d)", rack_name, rack_type_db, new_rack.id)
|
|
return rack_id_mapping
|
|
|
|
|
|
async def _upsert_infra_devices(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
infra_devices: list[tuple],
|
|
result: DiscoveryResult,
|
|
now: datetime,
|
|
) -> None:
|
|
for infra_ip, infra_category, infra_hostname, infra_verified in infra_devices:
|
|
try:
|
|
existing = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == infra_ip)
|
|
)).scalar_one_or_none()
|
|
|
|
if existing is None:
|
|
db.add(Device(
|
|
object_id=obj.id, ip=infra_ip, hostname=infra_hostname,
|
|
category=infra_category, role="other", source="auto_ssh",
|
|
location="server_room",
|
|
status="online" if infra_verified else "unknown",
|
|
last_seen=now if infra_verified else None,
|
|
))
|
|
logger.info(
|
|
"[discovery] %s (%s) - ADDED: category=%s, status=%s",
|
|
infra_ip, infra_hostname, infra_category,
|
|
"online" if infra_verified else "unknown",
|
|
)
|
|
result.created += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=infra_ip, hostname=infra_hostname, action="ADDED",
|
|
reason=f"object infrastructure device ({infra_category})",
|
|
category=infra_category, role="other",
|
|
))
|
|
else:
|
|
changes = []
|
|
if infra_verified:
|
|
if existing.status != "online":
|
|
existing.status = "online"
|
|
changes.append("status: →online")
|
|
existing.last_seen = now
|
|
changes.append(f"last_seen: →{now.isoformat()}")
|
|
if changes:
|
|
logger.info("[discovery] %s (%s) - UPDATED: %s", infra_ip, infra_hostname, ", ".join(changes))
|
|
result.updated += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=infra_ip, hostname=existing.hostname or infra_hostname,
|
|
action="UPDATED", reason="; ".join(changes),
|
|
category=existing.category, role=existing.role,
|
|
))
|
|
else:
|
|
logger.info("[discovery] %s (%s) - SKIPPED: already up to date", infra_ip, infra_hostname)
|
|
result.skipped += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=infra_ip, hostname=existing.hostname or infra_hostname,
|
|
action="SKIPPED", reason=f"already exists (category={existing.category})",
|
|
category=existing.category, role=existing.role,
|
|
))
|
|
except Exception as exc:
|
|
logger.error("[discovery] %s (%s) - ERROR: %s", infra_ip, infra_hostname, exc)
|
|
result.errors.append(f"Error upserting infra device {infra_ip}: {exc}")
|
|
|
|
if infra_devices:
|
|
await db.flush()
|
|
|
|
|
|
async def _resolve_rack_config_path(
|
|
db: AsyncSession, obj: Object, rack_host: str, default_path: str
|
|
) -> str:
|
|
"""Return the config path to use for this rack, honouring any per-device override."""
|
|
existing = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == rack_host)
|
|
)).scalar_one_or_none()
|
|
if existing and existing.connection_params:
|
|
return existing.connection_params.get("config_path", default_path)
|
|
return default_path
|
|
|
|
|
|
async def _ssh_fetch_rack_config(
|
|
rack_host: str,
|
|
ssh_port: int,
|
|
ssh_options: list,
|
|
config_path: str,
|
|
rack_name: str,
|
|
result: DiscoveryResult,
|
|
_emit,
|
|
) -> tuple[bool, str | None]:
|
|
"""Attempt to SSH into rack_host and read its config file.
|
|
|
|
Returns (ssh_ok, config_text).
|
|
"""
|
|
await _emit("disc_action", {"message": f"SSH → {rack_host} ({rack_name})"})
|
|
try:
|
|
config_text = await ssh_run_multi(rack_host, ssh_port, ssh_options, f"cat {config_path}")
|
|
return True, config_text
|
|
except Exception as exc:
|
|
result.errors.append(f"SSH to {rack_host} failed: {exc}")
|
|
return False, None
|
|
|
|
|
|
async def _upsert_rack_device(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
external_id: str,
|
|
rack_host: str,
|
|
rack_name: str,
|
|
rack_id_mapping: dict,
|
|
rack_ssh_ok: bool,
|
|
now: datetime,
|
|
result: DiscoveryResult,
|
|
) -> None:
|
|
"""Upsert the rack PC itself as an embedded device."""
|
|
rack_hostname = external_id
|
|
logger.info(
|
|
"[discovery] Processing rack %s (%s), ssh_ok=%s",
|
|
rack_host, rack_hostname, rack_ssh_ok,
|
|
)
|
|
try:
|
|
existing = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == rack_host)
|
|
)).scalar_one_or_none()
|
|
|
|
if existing is None:
|
|
status = "online" if rack_ssh_ok else "unknown"
|
|
db.add(Device(
|
|
object_id=obj.id, ip=rack_host, hostname=rack_hostname,
|
|
category="embedded", role="other", source="auto_ssh",
|
|
rack_id=rack_id_mapping.get(external_id),
|
|
status=status, last_seen=now if rack_ssh_ok else None,
|
|
))
|
|
logger.info("[discovery] %s (%s) - ADDED: status=%s", rack_host, rack_hostname, status)
|
|
result.created += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=rack_host, hostname=rack_hostname, action="ADDED",
|
|
reason="rack PC from racks table" + (" (SSH ok)" if rack_ssh_ok else " (SSH failed, added as unknown)"),
|
|
category="embedded", role="other",
|
|
))
|
|
else:
|
|
changes = []
|
|
if rack_ssh_ok:
|
|
if existing.status != "online":
|
|
existing.status = "online"
|
|
changes.append("status: →online")
|
|
existing.last_seen = now
|
|
changes.append("last_seen: updated")
|
|
if existing.rack_id is None and rack_id_mapping.get(external_id) is not None:
|
|
existing.rack_id = rack_id_mapping[external_id]
|
|
changes.append(f"rack_id: None→{existing.rack_id}")
|
|
if changes:
|
|
logger.info("[discovery] %s (%s) - UPDATED: %s", rack_host, existing.hostname, ", ".join(changes))
|
|
result.updated += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=rack_host, hostname=existing.hostname, action="UPDATED",
|
|
reason="; ".join(changes), category=existing.category, role=existing.role,
|
|
))
|
|
else:
|
|
reason = "SSH failed, rack already in DB" if not rack_ssh_ok else "rack already up to date"
|
|
logger.info("[discovery] %s (%s) - SKIPPED: %s", rack_host, existing.hostname, reason)
|
|
result.skipped += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=rack_host, hostname=existing.hostname, action="SKIPPED",
|
|
reason=reason, category=existing.category, role=existing.role,
|
|
))
|
|
except Exception as exc:
|
|
logger.error("[discovery] %s - ERROR: %s", rack_host, exc)
|
|
result.errors.append(f"Error processing rack {rack_host}: {exc}")
|
|
result.actions.append(DeviceAction(
|
|
ip=rack_host, hostname=rack_hostname, action="ERROR", reason=str(exc),
|
|
category="embedded", role="other",
|
|
))
|
|
|
|
|
|
async def _upsert_plugin_device(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
found: DiscoveredDevice,
|
|
external_id: str,
|
|
rack_id_mapping: dict,
|
|
result: DiscoveryResult,
|
|
slave_rack_id: int | None = None,
|
|
slave_identifier: str | None = None,
|
|
) -> None:
|
|
"""Upsert a single device discovered from a plugin section.
|
|
|
|
Works for both primary rack devices and slave rack devices.
|
|
When slave_rack_id / slave_identifier are given, the device is treated as
|
|
belonging to that slave rack instead of the primary rack keyed by external_id.
|
|
"""
|
|
is_slave = slave_identifier is not None
|
|
is_shared = found.category == "vm"
|
|
|
|
if is_shared:
|
|
hostname = found.plugin_name
|
|
rack_id: int | None = None
|
|
device_location: str | None = "server_room"
|
|
elif is_slave:
|
|
hostname = f"{slave_identifier}-{found.plugin_name}"
|
|
rack_id = slave_rack_id
|
|
device_location = None
|
|
else:
|
|
hostname = f"{external_id}-{found.plugin_name}"
|
|
rack_id = rack_id_mapping.get(external_id)
|
|
device_location = None
|
|
|
|
try:
|
|
existing = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == found.ip)
|
|
)).scalar_one_or_none()
|
|
|
|
if existing is not None:
|
|
# Protect embedded (rack PC) devices from being overwritten by plugin devices.
|
|
if existing.category == "embedded":
|
|
logger.info(
|
|
"[discovery] %s (%s) - SKIPPED: device is rack server (embedded)",
|
|
found.ip, found.plugin_name,
|
|
)
|
|
result.skipped += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=found.ip, hostname=existing.hostname, action="SKIPPED",
|
|
reason="device is rack server (embedded), cannot overwrite with plugin device",
|
|
category=existing.category, role=existing.role,
|
|
))
|
|
return
|
|
|
|
changes = []
|
|
if existing.category != found.category and existing.source == "auto_ssh":
|
|
changes.append(f"category: {existing.category}→{found.category}")
|
|
existing.category = found.category
|
|
if existing.role == "other" and found.role != "other":
|
|
changes.append(f"role: other→{found.role}")
|
|
existing.role = found.role
|
|
if not is_shared:
|
|
if not existing.hostname or existing.hostname.startswith("rack"):
|
|
if existing.hostname != hostname:
|
|
changes.append(f"hostname: '{existing.hostname}'→'{hostname}'")
|
|
existing.hostname = hostname
|
|
if existing.rack_id is None and rack_id is not None:
|
|
existing.rack_id = rack_id
|
|
changes.append(f"rack_id: None→{rack_id}")
|
|
|
|
if changes:
|
|
logger.info("[discovery] %s (%s) - UPDATED: %s", found.ip, found.plugin_name, ", ".join(changes))
|
|
result.updated += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=found.ip, hostname=existing.hostname, action="UPDATED",
|
|
reason="; ".join(changes), category=found.category, role=found.role,
|
|
))
|
|
else:
|
|
logger.info(
|
|
"[discovery] %s (%s) - SKIPPED: already exists with same values",
|
|
found.ip, found.plugin_name,
|
|
)
|
|
result.skipped += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=found.ip, hostname=existing.hostname, action="SKIPPED",
|
|
reason=f"already exists (source={existing.source}, category={existing.category})",
|
|
category=existing.category, role=existing.role,
|
|
))
|
|
else:
|
|
conn_params: dict = {}
|
|
if found.http_username:
|
|
conn_params["username"] = found.http_username
|
|
conn_params["password_enc"] = encrypt(found.http_password) if found.http_password else ""
|
|
db.add(Device(
|
|
object_id=obj.id, ip=found.ip, hostname=hostname,
|
|
category=found.category, role=found.role, source="auto_ssh",
|
|
rack_id=rack_id, location=device_location,
|
|
device_meta={"plugin": found.plugin_name, "cls": found.cls},
|
|
connection_params=conn_params,
|
|
))
|
|
cred_note = f", http_user={found.http_username}" if found.http_username else ""
|
|
logger.info(
|
|
"[discovery] %s (%s) - ADDED: category=%s, role=%s, rack_id=%s%s",
|
|
found.ip, found.plugin_name, found.category, found.role, rack_id, cred_note,
|
|
)
|
|
result.created += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=found.ip, hostname=hostname, action="ADDED",
|
|
reason=f"new device from plugin {found.plugin_name}",
|
|
category=found.category, role=found.role,
|
|
))
|
|
except Exception as exc:
|
|
logger.error("[discovery] %s (%s) - ERROR: %s", found.ip, found.plugin_name, exc)
|
|
result.errors.append(f"Error upserting {found.ip}: {exc}")
|
|
result.actions.append(DeviceAction(
|
|
ip=found.ip, hostname=hostname, action="ERROR", reason=str(exc),
|
|
category=found.category, role=found.role,
|
|
))
|
|
|
|
|
|
async def _discover_slave_racks(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
slave_candidates: set[str],
|
|
rack_hosts: list[tuple],
|
|
ssh_port: int,
|
|
ssh_options: list,
|
|
config_path: str,
|
|
all_rack_ips_set: set[str],
|
|
now: datetime,
|
|
result: DiscoveryResult,
|
|
_emit,
|
|
) -> None:
|
|
for candidate_ip in slave_candidates:
|
|
logger.info("[discovery] Checking slave candidate %s...", candidate_ip)
|
|
await _emit("disc_action", {"message": f"Проверяем слейв-кандидата {candidate_ip}..."})
|
|
|
|
slave_config = await fetch_slave_config(candidate_ip, ssh_port, ssh_options, config_path)
|
|
is_slave = False
|
|
slave_identifier = ""
|
|
master_ip_for_slave = ""
|
|
|
|
if slave_config is None:
|
|
logger.info("[discovery] %s — could not read config, skipping", candidate_ip)
|
|
else:
|
|
for _, master_host, _, _ in rack_hosts:
|
|
ok, _, _ = parse_slave_check(slave_config, master_host, candidate_ip)
|
|
if ok:
|
|
is_slave = True
|
|
master_ip_for_slave = master_host
|
|
master_ext_id = next(
|
|
(ext_id for ext_id, h, _, _ in rack_hosts if h == master_host),
|
|
master_host,
|
|
)
|
|
slave_identifier = f"{master_ext_id}-slave"
|
|
break
|
|
|
|
if not is_slave:
|
|
logger.info("[discovery] %s — not a slave rack", candidate_ip)
|
|
continue
|
|
|
|
logger.info("[discovery] %s — SLAVE confirmed (name=%s, master=%s)", candidate_ip, slave_identifier, master_ip_for_slave)
|
|
await _emit("disc_action", {"message": f"Слейв подтверждён: {slave_identifier} ({candidate_ip})"})
|
|
|
|
slave_rack_id = await _ensure_slave_rack(db, obj, slave_identifier, result)
|
|
|
|
await _upsert_slave_device(db, obj, candidate_ip, slave_identifier, slave_rack_id, master_ip_for_slave, now, result)
|
|
|
|
all_rack_ips_with_slave = list(all_rack_ips_set) + [candidate_ip]
|
|
slave_devices = parse_config(slave_config, candidate_ip, all_rack_ips_with_slave)
|
|
logger.info("[discovery] Slave %s: found %d plugin devices", candidate_ip, len(slave_devices))
|
|
await _emit("disc_slave_found", {
|
|
"rack_name": slave_identifier, "ip": candidate_ip, "found": len(slave_devices)
|
|
})
|
|
|
|
for found in slave_devices:
|
|
await _upsert_plugin_device(
|
|
db, obj, found,
|
|
external_id="", # not used for slaves (is_slave=True path)
|
|
rack_id_mapping={},
|
|
result=result,
|
|
slave_rack_id=slave_rack_id,
|
|
slave_identifier=slave_identifier,
|
|
)
|
|
|
|
try:
|
|
await db.flush()
|
|
except Exception as exc:
|
|
result.errors.append(f"Flush error after slave {candidate_ip}: {exc}")
|
|
|
|
|
|
async def _ensure_slave_rack(
|
|
db: AsyncSession, obj: Object, slave_identifier: str, result: DiscoveryResult
|
|
) -> int | None:
|
|
"""Ensure a Rack entry exists for the slave and return its ID."""
|
|
try:
|
|
existing = (await db.execute(
|
|
select(Rack).where(Rack.object_id == obj.id, Rack.name == slave_identifier)
|
|
)).scalar_one_or_none()
|
|
if existing:
|
|
return existing.id
|
|
new_rack = Rack(
|
|
object_id=obj.id, name=slave_identifier,
|
|
description="slave (auto-discovered via ARP)",
|
|
)
|
|
db.add(new_rack)
|
|
await db.flush()
|
|
logger.info("[discovery] Created rack '%s' (id=%d) for slave", slave_identifier, new_rack.id)
|
|
return new_rack.id
|
|
except Exception as exc:
|
|
result.errors.append(f"Error creating rack entry for slave: {exc}")
|
|
return None
|
|
|
|
|
|
async def _upsert_slave_device(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
candidate_ip: str,
|
|
slave_identifier: str,
|
|
slave_rack_id: int | None,
|
|
master_ip: str,
|
|
now: datetime,
|
|
result: DiscoveryResult,
|
|
) -> None:
|
|
try:
|
|
existing = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == candidate_ip)
|
|
)).scalar_one_or_none()
|
|
if existing is None:
|
|
db.add(Device(
|
|
object_id=obj.id, ip=candidate_ip, hostname=slave_identifier,
|
|
category="embedded", role="other", source="auto_ssh",
|
|
rack_id=slave_rack_id, status="online", last_seen=now,
|
|
))
|
|
logger.info("[discovery] %s (%s) - ADDED: slave rack PC", candidate_ip, slave_identifier)
|
|
result.created += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=candidate_ip, hostname=slave_identifier, action="ADDED",
|
|
reason=f"slave rack PC (master={master_ip})",
|
|
category="embedded", role="other",
|
|
))
|
|
else:
|
|
changes = []
|
|
if existing.status != "online":
|
|
existing.status = "online"
|
|
changes.append("status: →online")
|
|
existing.last_seen = now
|
|
changes.append("last_seen: updated")
|
|
if existing.rack_id is None and slave_rack_id:
|
|
existing.rack_id = slave_rack_id
|
|
changes.append(f"rack_id: →{slave_rack_id}")
|
|
result.updated += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=candidate_ip, hostname=existing.hostname or slave_identifier,
|
|
action="UPDATED", reason="; ".join(changes),
|
|
category="embedded", role="other",
|
|
))
|
|
await db.flush()
|
|
except Exception as exc:
|
|
result.errors.append(f"Error upserting slave device {candidate_ip}: {exc}")
|
|
|
|
|
|
async def _discover_mikrotik(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
rack_hosts: list[tuple],
|
|
ssh_port: int,
|
|
ssh_options: list,
|
|
result: DiscoveryResult,
|
|
_emit,
|
|
) -> None:
|
|
target_host = obj.server_ip or rack_hosts[0][1]
|
|
logger.info("[discovery] Phase 3: MikroTik via SSH_CLIENT on %s", target_host)
|
|
mikrotik_ip = await discover_mikrotik_multi(target_host, ssh_port, ssh_options)
|
|
|
|
if not mikrotik_ip:
|
|
logger.info("[discovery] MikroTik: no IP found via SSH_CLIENT")
|
|
return
|
|
|
|
logger.info("[discovery] MikroTik found at %s", mikrotik_ip)
|
|
await _emit("disc_infra_found", {"type": "mikrotik", "ip": mikrotik_ip})
|
|
try:
|
|
existing = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == mikrotik_ip)
|
|
)).scalar_one_or_none()
|
|
if existing is not None:
|
|
logger.info("[discovery] %s (mikrotik) - SKIPPED: already exists", mikrotik_ip)
|
|
result.skipped += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=mikrotik_ip, hostname="mikrotik", action="SKIPPED",
|
|
reason=f"already exists (source={existing.source}, category={existing.category})",
|
|
category=existing.category, role=existing.role,
|
|
))
|
|
else:
|
|
db.add(Device(
|
|
object_id=obj.id, ip=mikrotik_ip, hostname="mikrotik",
|
|
category="router", role="other", source="auto_ssh",
|
|
))
|
|
logger.info("[discovery] %s (mikrotik) - ADDED: category=router", mikrotik_ip)
|
|
result.created += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=mikrotik_ip, hostname="mikrotik", action="ADDED",
|
|
reason="discovered via SSH_CLIENT",
|
|
category="router", role="other",
|
|
))
|
|
except Exception as exc:
|
|
logger.error("[discovery] %s (mikrotik) - ERROR: %s", mikrotik_ip, exc)
|
|
result.errors.append(f"Error upserting {mikrotik_ip}: {exc}")
|
|
result.actions.append(DeviceAction(
|
|
ip=mikrotik_ip, hostname="mikrotik", action="ERROR", reason=str(exc),
|
|
category="router", role="other",
|
|
))
|
|
await db.flush()
|
|
|
|
|
|
async def _discover_proxmox(
|
|
db: AsyncSession,
|
|
obj: Object,
|
|
ssh_port: int,
|
|
ssh_password: str,
|
|
rack_hosts: list[tuple],
|
|
result: DiscoveryResult,
|
|
_emit,
|
|
) -> None:
|
|
target_host = obj.server_ip or rack_hosts[0][1]
|
|
logger.info("[discovery] Phase 4: Proxmox port 8006 scan on %s", target_host)
|
|
proxmox_ips = await discover_proxmox_via_port_scan(
|
|
host=target_host, port=obj.ssh_port or 22,
|
|
username=obj.ssh_user, password=ssh_password,
|
|
)
|
|
if not proxmox_ips:
|
|
logger.info("[discovery] Proxmox: no servers found on port 8006")
|
|
return
|
|
|
|
logger.info("[discovery] Proxmox: found %d server(s): %s", len(proxmox_ips), proxmox_ips)
|
|
for proxmox_ip in proxmox_ips:
|
|
await _emit("disc_infra_found", {"type": "proxmox", "ip": proxmox_ip})
|
|
|
|
for idx, proxmox_ip in enumerate(proxmox_ips, 1):
|
|
hostname = f"proxmox_{idx}"
|
|
try:
|
|
existing = (await db.execute(
|
|
select(Device).where(Device.object_id == obj.id, Device.ip == proxmox_ip)
|
|
)).scalar_one_or_none()
|
|
if existing is not None:
|
|
logger.info("[discovery] %s (%s) - SKIPPED: already exists", proxmox_ip, hostname)
|
|
result.skipped += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=proxmox_ip, hostname=hostname, action="SKIPPED",
|
|
reason=f"already exists (source={existing.source}, category={existing.category})",
|
|
category=existing.category, role=existing.role,
|
|
))
|
|
else:
|
|
db.add(Device(
|
|
object_id=obj.id, ip=proxmox_ip, hostname=hostname,
|
|
category="vm", role="other", source="auto_ssh",
|
|
))
|
|
logger.info("[discovery] %s (%s) - ADDED: category=vm", proxmox_ip, hostname)
|
|
result.created += 1
|
|
result.actions.append(DeviceAction(
|
|
ip=proxmox_ip, hostname=hostname, action="ADDED",
|
|
reason="discovered via port 8006 scan",
|
|
category="vm", role="other",
|
|
))
|
|
except Exception as exc:
|
|
logger.error("[discovery] %s (%s) - ERROR: %s", proxmox_ip, hostname, exc)
|
|
result.errors.append(f"Error upserting {proxmox_ip}: {exc}")
|
|
result.actions.append(DeviceAction(
|
|
ip=proxmox_ip, hostname=hostname, action="ERROR", reason=str(exc),
|
|
category="vm", role="other",
|
|
))
|
|
await db.flush()
|
|
|
|
|
|
# ── Summary logging ───────────────────────────────────────────────────────────
|
|
|
|
def _log_summary(result: DiscoveryResult) -> None:
|
|
sep = "=" * 100
|
|
logger.info(sep)
|
|
logger.info(
|
|
"[discovery] SUMMARY: ADDED=%d, UPDATED=%d, SKIPPED=%d, ERRORS=%d",
|
|
result.created, result.updated, result.skipped, len(result.errors),
|
|
)
|
|
by_action: dict[str, list[DeviceAction]] = {}
|
|
for action in result.actions:
|
|
by_action.setdefault(action.action, []).append(action)
|
|
|
|
for label, actions in by_action.items():
|
|
log_fn = logger.error if label == "ERROR" else logger.info
|
|
log_fn("[discovery] ─── %s devices ───", label)
|
|
for a in actions:
|
|
log_fn(" %-20s (%-30s) | %-15s | %-10s | %s", a.ip, a.hostname, a.category, a.role, a.reason)
|
|
logger.info(sep)
|