""" Main discovery orchestrator. Coordinates all phases of SSH-based device discovery: 0. Resolve DB connection (explicit config or autodetect via SSH). 0.5 Pre-upsert the DB server device (so users can configure tunnels on failure). 1. Fetch rack hosts from the CAPS DB; sync Rack table entries. 1b. Upsert infrastructure devices (main_server). 2. SSH into each rack → parse CAPS config → upsert plugin devices. 2.5 Discover slave racks via ARP neighbor tables. 3. Discover MikroTik via SSH_CLIENT env var. 4. Discover Proxmox via port 8006 scan. """ import logging from configparser import RawConfigParser from datetime import datetime, timezone from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.models.device import Device from app.models.object import Object from app.models.rack import Rack from app.services.crypto import decrypt from .config_parser import parse_config, parse_slave_check from .db_resolver import autodetect_db_from_server, get_rack_hosts from .ssh_client import ( discover_mikrotik_multi, discover_proxmox_via_port_scan, fetch_slave_config, get_ip_neighbors_multi, ssh_run_multi, verify_is_mikrotik, emit_ping_status, ) from .types import DeviceAction, DiscoveredDevice, DiscoveryResult, ProgressCallback logger = logging.getLogger(__name__) _LOCAL_DB_HOSTS = {"localhost", "127.0.0.1", "::1"} # ── Public entry point ──────────────────────────────────────────────────────── async def discover_via_ssh( db: AsyncSession, obj: Object, config_path: str = "/etc/caps/config.ini", progress_cb: ProgressCallback | None = None, ) -> DiscoveryResult: result = DiscoveryResult() async def _emit(event_type: str, data: dict) -> None: if progress_cb is not None: await progress_cb(event_type, data) from app.services.ssh import build_object_auth_options ssh_options = await build_object_auth_options(obj, db) if not ssh_options: result.errors.append("SSH credentials not configured on this object") return result ssh_port = obj.ssh_port or 22 embedded_ssh_ports = _embedded_ssh_ports(obj) # First option's password is used by single-auth Proxmox scan (legacy). _first = ssh_options[0] ssh_password = _first.password if _first.type == "password" else "" # ── Phase 0: Resolve DB connection ──────────────────────────────────────── db_params = await _resolve_db_connection(obj, ssh_port, ssh_options, result, _emit) if db_params is None: return result db_host, db_port, db_name, db_user, db_password, db_table, server_ip_ssh_verified = db_params # ── Phase 0.5: Pre-upsert DB server device ──────────────────────────────── now = datetime.now(timezone.utc) db_device_obj, use_tunnel = await _pre_upsert_db_device( db, obj, db_host, result, now ) # ── Phase 1: Fetch rack hosts ────────────────────────────────────────────── await _emit("disc_action", {"message": "Получение списка стоек из БД..."}) logger.info( "[discovery] Connecting to object DB: host=%s port=%s dbname=%s user=%s table=%s via_tunnel=%s", db_host, db_port, db_name, db_user, db_table, use_tunnel, ) try: rack_hosts = await get_rack_hosts( host=db_host, port=db_port, user=db_user, password=db_password or "", dbname=db_name, table=db_table, via_tunnel=use_tunnel, tunnel_host=obj.server_ip, tunnel_ssh_port=ssh_port, ssh_options=ssh_options, progress_cb=_emit, ) if db_device_obj is not None: db_device_obj.status = "online" db_device_obj.last_seen = now except Exception as exc: if db_device_obj is not None: db_device_obj.status = "offline" err_msg = f"Cannot fetch rack hosts from object DB: {exc}" if "timeout" in str(exc).lower(): err_msg += ". Попробуйте настроить SSH-туннель (отредактируйте устройство БД и включите опцию)" result.errors.append(err_msg) return result if not rack_hosts: result.errors.append("No racks with host IPs found in object DB") return result await _emit("disc_server", {"rack_count": len(rack_hosts)}) # ── Phase 1a: Sync Rack table entries ────────────────────────────────────── rack_id_mapping = await _sync_rack_entries(db, obj, rack_hosts) # ── Phase 1b: Upsert infrastructure devices ─────────────────────────────── infra_devices = [] if obj.server_ip: infra_devices.append((obj.server_ip, "main_server", "caps_server", server_ip_ssh_verified)) await _upsert_infra_devices(db, obj, infra_devices, result, now) # ── Phase 2: Process racks ──────────────────────────────────────────────── neighbor_candidates: set[str] = set() discovered_device_ips: set[str] = set() all_rack_ips = [h for _, h, _, _ in rack_hosts] for rack_idx, (external_id, rack_host, rack_name, _rack_type_db) in enumerate(rack_hosts): await _emit("disc_rack_start", { "rack_name": rack_name, "rack_host": rack_host, "index": rack_idx, "total": len(rack_hosts), }) _rack_found: int = 0 _rack_ssh_ok: bool = False _flush_error: Exception | None = None try: # Allow per-device config_path override stored in connection_params. effective_config_path = await _resolve_rack_config_path( db, obj, rack_host, config_path ) rack_ssh_ok, config_text, rack_ssh_port = await _ssh_fetch_rack_config( rack_host, embedded_ssh_ports, ssh_options, effective_config_path, rack_name, result, _emit ) _rack_ssh_ok = rack_ssh_ok await _upsert_rack_device( db, obj, external_id, rack_host, rack_name, rack_id_mapping, rack_ssh_ok, rack_ssh_port, now, result ) if rack_ssh_ok: neigh_ips = await get_ip_neighbors_multi( rack_host, rack_ssh_port or embedded_ssh_ports[0], ssh_options, progress_cb=_emit ) neighbor_candidates.update(neigh_ips) discovered_device_ips.add(rack_host) await _emit("disc_action", {"message": f"Парсинг конфигурации {rack_name}..."}) devices_found = parse_config(config_text, rack_host, all_rack_ips, obj.server_ip) logger.info("[discovery] Found %d devices from plugins in %s", len(devices_found), rack_host) _rack_found = len(devices_found) for found in devices_found: ping_online, ping_ms = await emit_ping_status( found.ip, _emit, phase="discovered_device" ) await _upsert_plugin_device( db, obj, found, external_id, rack_id_mapping, result, ping_online=ping_online, ping_ms=ping_ms if ping_online else None, checked_at=now, ) discovered_device_ips.add(found.ip) try: await db.flush() except Exception as exc: # Session is now invalid after a failed flush — capture the error so we can # still emit disc_rack_done before propagating and aborting discovery. _flush_error = exc except Exception as exc: # Unexpected per-rack error (e.g. SSH timeout, config parse failure not caught # at a lower level). Record it and fall through to emit disc_rack_done. result.errors.append(f"Rack {rack_name} ({rack_host}): {exc}") logger.error("[discovery] Rack %s (%s) error: %s", rack_host, rack_name, exc, exc_info=True) # disc_rack_done is always emitted — frontend must never be left with a spinner # for a rack that already received disc_rack_start. await _emit("disc_rack_done", { "rack_name": rack_name, "rack_host": rack_host, "found": _rack_found, "ssh_ok": _rack_ssh_ok, }) if _flush_error: raise RuntimeError(f"DB flush failed after rack {rack_host}: {_flush_error}") from _flush_error # ── Phase 2.5: Slave rack discovery via ARP ─────────────────────────────── all_rack_ips_set = {h for _, h, _, _ in rack_hosts} infra_ips = {ip for ip in [obj.server_ip, db_host] if ip} common_gateways = {ip for ip in neighbor_candidates if ip.endswith(".1") or ip.endswith(".254")} slave_candidates = ( neighbor_candidates - all_rack_ips_set - discovered_device_ips - infra_ips - common_gateways ) await _emit("disc_phase", {"phase": "slaves", "candidate_count": len(slave_candidates)}) if slave_candidates: logger.info("[discovery] Phase 2.5: Slave discovery — %d candidate(s)", len(slave_candidates)) await _emit("disc_action", { "message": f"Поиск слейв-стоек: проверяем {len(slave_candidates)} ARP-кандидатов..." }) else: await _emit("disc_action", {"message": "Слейв-стойки: ARP-кандидатов не найдено"}) await _discover_slave_racks( db, obj, slave_candidates, rack_hosts, embedded_ssh_ports, ssh_options, config_path, all_rack_ips_set, now, result, _emit ) # ── Phase 3: MikroTik via SSH_CLIENT ────────────────────────────────────── await _emit("disc_phase", {"phase": "mikrotik"}) await _emit("disc_action", {"message": "Поиск MikroTik (SSH_CLIENT)..."}) await _discover_mikrotik(db, obj, rack_hosts, ssh_port, ssh_options, result, _emit) # ── Phase 4: Proxmox via port 8006 scan ─────────────────────────────────── await _emit("disc_phase", {"phase": "proxmox"}) await _emit("disc_action", {"message": "Сканирование порта 8006 (Proxmox)..."}) await _discover_proxmox(db, obj, ssh_port, ssh_password, rack_hosts, result, _emit) # ── Summary ─────────────────────────────────────────────────────────────── _log_summary(result) return result # ── Phase helpers ───────────────────────────────────────────────────────────── def _embedded_ssh_ports(obj: Object) -> list[int]: """Return SSH ports to try for embedded rack PCs.""" ports = [22] object_port = obj.ssh_port or 22 if object_port not in ports: ports.append(object_port) return ports def _normalize_db_host(db_host: str | None, server_ip: str | None) -> str | None: """Replace localhost-like DB hosts with object server IP for remote discovery.""" if not db_host: return db_host host = db_host.strip() if host.lower() in _LOCAL_DB_HOSTS and server_ip: logger.info( "[discovery] DB host '%s' mapped to object server_ip '%s'", host, server_ip, ) return server_ip return host async def _resolve_db_connection( obj: Object, ssh_port: int, ssh_options: list, result: DiscoveryResult, _emit, ) -> tuple | None: """Determine the DB connection parameters. Returns (db_host, db_port, db_name, db_user, db_password, db_table, server_ip_ssh_verified) or None if resolution failed (errors added to result). """ db_host = obj.db_host db_port = obj.db_port or 5432 db_name = obj.db_name db_user = obj.db_user db_password: str | None = None db_table = obj.db_table server_ip_ssh_verified = False if db_host and db_name and db_user and obj.db_pass_enc and db_table: try: db_password = decrypt(obj.db_pass_enc) except Exception: result.errors.append("Failed to decrypt DB password") return None db_host = _normalize_db_host(db_host, obj.server_ip) elif obj.server_ip: await _emit("disc_action", {"message": f"Чтение конфигурации сервера {obj.server_ip}..."}) try: detected = await autodetect_db_from_server( server_ip=obj.server_ip, ssh_port=ssh_port, ssh_options=ssh_options, progress_cb=_emit, ) except Exception as exc: result.errors.append( f"Could not read server config for auto-detection: {exc}. " f"Fill in DB fields manually on the object." ) return None if not detected or not all([detected.get("host"), detected.get("user"), detected.get("dbname")]): await _emit_db_autodetect_error(obj, ssh_port, ssh_options, result) return None db_host = detected["host"] db_host = _normalize_db_host(db_host, obj.server_ip) db_port = detected["port"] db_user = detected["user"] db_password = detected["password"] db_name = detected["dbname"] db_table = db_table or obj.db_table or "racks" server_ip_ssh_verified = True else: result.errors.append( "DB connection not configured. Either fill in the DB fields on the object " "or set server_ip so the DB can be auto-detected via SSH." ) return None return db_host, db_port, db_name, db_user, db_password, db_table, server_ip_ssh_verified async def _emit_db_autodetect_error(obj, ssh_port, ssh_options, result: DiscoveryResult) -> None: try: raw = await ssh_run_multi( obj.server_ip, ssh_port, ssh_options, "cat /etc/caps/config.ini", timeout=30, progress_cb=None, ) diag_parser = RawConfigParser(strict=False) diag_parser.read_string(raw) found_sections = diag_parser.sections() found_keys = {s: list(diag_parser.options(s)) for s in found_sections} found_db_uri_sections = [ s for s in found_sections if diag_parser.get(s, "db_uri", fallback=None) ] db_uri_note = ( f" db_uri keys were found in sections {found_db_uri_sections}, " f"but none could be parsed as a PostgreSQL URI." if found_db_uri_sections else "" ) result.errors.append( f"db_uri not found in server config. " f"{db_uri_note}" f"Sections found: {found_sections}. " f"Keys per section: {found_keys}. " f"Fill in DB fields manually on the object." ) except Exception as diag_exc: result.errors.append( f"Could not read server config for auto-detection: {diag_exc}. " f"Fill in DB fields manually on the object." ) async def _pre_upsert_db_device( db: AsyncSession, obj: Object, db_host: str | None, result: DiscoveryResult, now: datetime, ) -> tuple: """Upsert the CAPS DB server as a device before the DB connection attempt. Returns (db_device_obj, use_tunnel). """ db_device_obj: Device | None = None if db_host and db_host != obj.server_ip: try: db_device_obj = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == db_host) )).scalar_one_or_none() if db_device_obj is None: db_device_obj = Device( object_id=obj.id, ip=db_host, hostname="db_server", category="vm", role="other", source="auto_ssh", location="server_room", status="unknown", device_meta={"is_caps_db": True}, ) db.add(db_device_obj) await db.flush() logger.info("[discovery] %s (db_server) - PRE-ADDED before connection attempt", db_host) result.created += 1 result.actions.append(DeviceAction( ip=db_host, hostname="db_server", action="ADDED", reason="object infrastructure device (vm/db_server)", category="vm", role="other", )) except Exception as exc: logger.warning("[discovery] Failed to pre-upsert db_server %s: %s", db_host, exc) device_via_tunnel = bool( (db_device_obj.connection_params or {}).get("db_via_tunnel") ) if db_device_obj else False use_tunnel = device_via_tunnel or obj.db_via_tunnel return db_device_obj, use_tunnel async def _sync_rack_entries( db: AsyncSession, obj: Object, rack_hosts: list[tuple], ) -> dict[str, int]: """Ensure a Rack row exists for every rack returned from the CAPS DB. Returns a mapping of external_id → Rack.id. """ rack_id_mapping: dict[str, int] = {} for ext_id, _, rack_name, rack_type_db in rack_hosts: existing = (await db.execute( select(Rack).where( Rack.object_id == obj.id, Rack.name.in_([ext_id, rack_name]), ) )).scalar_one_or_none() if existing: rack_id_mapping[ext_id] = existing.id if existing.name != rack_name: logger.info("[discovery] Renamed rack '%s' → '%s'", existing.name, rack_name) existing.name = rack_name if rack_type_db and not existing.rack_type: existing.rack_type = rack_type_db else: new_rack = Rack(object_id=obj.id, name=rack_name, rack_type=rack_type_db) db.add(new_rack) await db.flush() rack_id_mapping[ext_id] = new_rack.id logger.info("[discovery] Created rack '%s' rack_type=%s (id=%d)", rack_name, rack_type_db, new_rack.id) return rack_id_mapping async def _upsert_infra_devices( db: AsyncSession, obj: Object, infra_devices: list[tuple], result: DiscoveryResult, now: datetime, ) -> None: for infra_ip, infra_category, infra_hostname, infra_verified in infra_devices: try: existing = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == infra_ip) )).scalar_one_or_none() if existing is None: db.add(Device( object_id=obj.id, ip=infra_ip, hostname=infra_hostname, category=infra_category, role="other", source="auto_ssh", location="server_room", status="online" if infra_verified else "unknown", last_seen=now if infra_verified else None, )) logger.info( "[discovery] %s (%s) - ADDED: category=%s, status=%s", infra_ip, infra_hostname, infra_category, "online" if infra_verified else "unknown", ) result.created += 1 result.actions.append(DeviceAction( ip=infra_ip, hostname=infra_hostname, action="ADDED", reason=f"object infrastructure device ({infra_category})", category=infra_category, role="other", )) else: changes = [] if infra_verified: if existing.status != "online": existing.status = "online" changes.append("status: →online") existing.last_seen = now changes.append(f"last_seen: →{now.isoformat()}") if changes: logger.info("[discovery] %s (%s) - UPDATED: %s", infra_ip, infra_hostname, ", ".join(changes)) result.updated += 1 result.actions.append(DeviceAction( ip=infra_ip, hostname=existing.hostname or infra_hostname, action="UPDATED", reason="; ".join(changes), category=existing.category, role=existing.role, )) else: logger.info("[discovery] %s (%s) - SKIPPED: already up to date", infra_ip, infra_hostname) result.skipped += 1 result.actions.append(DeviceAction( ip=infra_ip, hostname=existing.hostname or infra_hostname, action="SKIPPED", reason=f"already exists (category={existing.category})", category=existing.category, role=existing.role, )) except Exception as exc: logger.error("[discovery] %s (%s) - ERROR: %s", infra_ip, infra_hostname, exc) result.errors.append(f"Error upserting infra device {infra_ip}: {exc}") if infra_devices: await db.flush() async def _resolve_rack_config_path( db: AsyncSession, obj: Object, rack_host: str, default_path: str ) -> str: """Return the config path to use for this rack, honouring any per-device override.""" existing = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == rack_host) )).scalar_one_or_none() if existing and existing.connection_params: return existing.connection_params.get("config_path", default_path) return default_path async def _ssh_fetch_rack_config( rack_host: str, ssh_ports: list[int], ssh_options: list, config_path: str, rack_name: str, result: DiscoveryResult, _emit, ) -> tuple[bool, str | None, int | None]: """Attempt to SSH into rack_host and read its config file. Returns (ssh_ok, config_text, ssh_port_used). """ await _emit("disc_action", {"message": f"SSH -> {rack_host} ({rack_name})"}) last_exc: Exception | None = None for ssh_port in ssh_ports: for attempt in range(1, 3): try: config_text = await ssh_run_multi( rack_host, ssh_port, ssh_options, f"cat {config_path}", timeout=30, progress_cb=_emit, ) return True, config_text, ssh_port except TimeoutError as exc: last_exc = exc if attempt < 2: logger.warning( "[discovery] Reading rack config from %s:%d timed out; retrying (%d/2)", rack_host, ssh_port, attempt + 1, ) await _emit("disc_action", { "message": f"Retrying config read {rack_host}:{ssh_port} ({rack_name})..." }) continue break except Exception as exc: last_exc = exc logger.info("[discovery] SSH to rack %s:%d failed: %s", rack_host, ssh_port, exc) break result.errors.append(f"SSH to {rack_host} failed on ports {ssh_ports}: {last_exc}") return False, None, None async def _upsert_rack_device( db: AsyncSession, obj: Object, external_id: str, rack_host: str, rack_name: str, rack_id_mapping: dict, rack_ssh_ok: bool, rack_ssh_port: int | None, now: datetime, result: DiscoveryResult, ) -> None: """Upsert the rack PC itself as an embedded device.""" rack_hostname = external_id logger.info( "[discovery] Processing rack %s (%s), ssh_ok=%s", rack_host, rack_hostname, rack_ssh_ok, ) rack_port_override = ( rack_ssh_port if rack_ssh_ok and rack_ssh_port and rack_ssh_port != (obj.ssh_port or 22) else None ) try: existing = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == rack_host) )).scalar_one_or_none() if existing is None: status = "online" if rack_ssh_ok else "unknown" db.add(Device( object_id=obj.id, ip=rack_host, hostname=rack_hostname, category="embedded", role="other", source="auto_ssh", rack_id=rack_id_mapping.get(external_id), status=status, last_seen=now if rack_ssh_ok else None, ssh_port_override=rack_port_override, )) logger.info("[discovery] %s (%s) - ADDED: status=%s", rack_host, rack_hostname, status) result.created += 1 result.actions.append(DeviceAction( ip=rack_host, hostname=rack_hostname, action="ADDED", reason="rack PC from racks table" + (" (SSH ok)" if rack_ssh_ok else " (SSH failed, added as unknown)"), category="embedded", role="other", )) else: changes = [] if rack_ssh_ok: if existing.status != "online": existing.status = "online" changes.append("status: →online") existing.last_seen = now changes.append("last_seen: updated") if existing.ssh_port_override != rack_port_override: existing.ssh_port_override = rack_port_override changes.append(f"ssh_port_override: ->{rack_port_override or 'default'}") if existing.rack_id is None and rack_id_mapping.get(external_id) is not None: existing.rack_id = rack_id_mapping[external_id] changes.append(f"rack_id: None→{existing.rack_id}") if changes: logger.info("[discovery] %s (%s) - UPDATED: %s", rack_host, existing.hostname, ", ".join(changes)) result.updated += 1 result.actions.append(DeviceAction( ip=rack_host, hostname=existing.hostname, action="UPDATED", reason="; ".join(changes), category=existing.category, role=existing.role, )) else: reason = "SSH failed, rack already in DB" if not rack_ssh_ok else "rack already up to date" logger.info("[discovery] %s (%s) - SKIPPED: %s", rack_host, existing.hostname, reason) result.skipped += 1 result.actions.append(DeviceAction( ip=rack_host, hostname=existing.hostname, action="SKIPPED", reason=reason, category=existing.category, role=existing.role, )) except Exception as exc: logger.error("[discovery] %s - ERROR: %s", rack_host, exc) result.errors.append(f"Error processing rack {rack_host}: {exc}") result.actions.append(DeviceAction( ip=rack_host, hostname=rack_hostname, action="ERROR", reason=str(exc), category="embedded", role="other", )) async def _upsert_plugin_device( db: AsyncSession, obj: Object, found: DiscoveredDevice, external_id: str, rack_id_mapping: dict, result: DiscoveryResult, slave_rack_id: int | None = None, slave_identifier: str | None = None, ping_online: bool | None = None, ping_ms: int | None = None, checked_at: datetime | None = None, ) -> None: """Upsert a single device discovered from a plugin section. Works for both primary rack devices and slave rack devices. When slave_rack_id / slave_identifier are given, the device is treated as belonging to that slave rack instead of the primary rack keyed by external_id. """ is_slave = slave_identifier is not None is_shared = found.category == "vm" if is_shared: hostname = found.plugin_name rack_id: int | None = None device_location: str | None = "server_room" elif is_slave: hostname = f"{slave_identifier}-{found.plugin_name}" rack_id = slave_rack_id device_location = None else: hostname = f"{external_id}-{found.plugin_name}" rack_id = rack_id_mapping.get(external_id) device_location = None try: existing = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == found.ip) )).scalar_one_or_none() if existing is not None: # Protect embedded (rack PC) devices from being overwritten by plugin devices. if existing.category == "embedded": logger.info( "[discovery] %s (%s) - SKIPPED: device is rack server (embedded)", found.ip, found.plugin_name, ) result.skipped += 1 result.actions.append(DeviceAction( ip=found.ip, hostname=existing.hostname, action="SKIPPED", reason="device is rack server (embedded), cannot overwrite with plugin device", category=existing.category, role=existing.role, )) return changes = [] if existing.category != found.category and existing.source == "auto_ssh": changes.append(f"category: {existing.category}→{found.category}") existing.category = found.category if existing.role == "other" and found.role != "other": changes.append(f"role: other→{found.role}") existing.role = found.role if not is_shared: if not existing.hostname or existing.hostname.startswith("rack"): if existing.hostname != hostname: changes.append(f"hostname: '{existing.hostname}'→'{hostname}'") existing.hostname = hostname if existing.rack_id is None and rack_id is not None: existing.rack_id = rack_id changes.append(f"rack_id: None→{rack_id}") # Always sync credentials from config — they may have changed if found.http_username: cp = dict(existing.connection_params or {}) if cp.get("username") != found.http_username or cp.get("password") != found.http_password: cp["username"] = found.http_username cp["password"] = found.http_password # plaintext — camera web auth existing.connection_params = cp changes.append(f"connection_params.credentials updated") if ping_online is not None: new_status = "online" if ping_online else "offline" if existing.status != new_status: existing.status = new_status changes.append(f"status: ->{new_status}") if checked_at is not None: existing.last_ping_at = checked_at if ping_online: existing.last_seen = checked_at existing.last_ping_ms = ping_ms if ping_online else None if changes: logger.info("[discovery] %s (%s) - UPDATED: %s", found.ip, found.plugin_name, ", ".join(changes)) result.updated += 1 result.actions.append(DeviceAction( ip=found.ip, hostname=existing.hostname, action="UPDATED", reason="; ".join(changes), category=found.category, role=found.role, )) else: logger.info( "[discovery] %s (%s) - SKIPPED: already exists with same values", found.ip, found.plugin_name, ) result.skipped += 1 result.actions.append(DeviceAction( ip=found.ip, hostname=existing.hostname, action="SKIPPED", reason=f"already exists (source={existing.source}, category={existing.category})", category=existing.category, role=existing.role, )) else: conn_params: dict = {} if found.http_username: conn_params["username"] = found.http_username conn_params["password"] = found.http_password # plaintext — camera web auth, not SSH db.add(Device( object_id=obj.id, ip=found.ip, hostname=hostname, category=found.category, role=found.role, source="auto_ssh", rack_id=rack_id, location=device_location, device_meta={"plugin": found.plugin_name, "cls": found.cls}, connection_params=conn_params, status="online" if ping_online else "offline" if ping_online is not None else "unknown", last_seen=checked_at if ping_online else None, last_ping_at=checked_at if ping_online is not None else None, last_ping_ms=ping_ms if ping_online else None, )) cred_note = f", http_user={found.http_username}" if found.http_username else "" logger.info( "[discovery] %s (%s) - ADDED: category=%s, role=%s, rack_id=%s%s", found.ip, found.plugin_name, found.category, found.role, rack_id, cred_note, ) result.created += 1 result.actions.append(DeviceAction( ip=found.ip, hostname=hostname, action="ADDED", reason=f"new device from plugin {found.plugin_name}", category=found.category, role=found.role, )) except Exception as exc: logger.error("[discovery] %s (%s) - ERROR: %s", found.ip, found.plugin_name, exc) result.errors.append(f"Error upserting {found.ip}: {exc}") result.actions.append(DeviceAction( ip=found.ip, hostname=hostname, action="ERROR", reason=str(exc), category=found.category, role=found.role, )) async def _discover_slave_racks( db: AsyncSession, obj: Object, slave_candidates: set[str], rack_hosts: list[tuple], ssh_ports: list[int], ssh_options: list, config_path: str, all_rack_ips_set: set[str], now: datetime, result: DiscoveryResult, _emit, ) -> None: for candidate_ip in slave_candidates: logger.info("[discovery] Checking slave candidate %s...", candidate_ip) await _emit("disc_action", {"message": f"Проверяем слейв-кандидата {candidate_ip}..."}) try: slave_config, slave_ssh_port = await fetch_slave_config( candidate_ip, ssh_ports, ssh_options, config_path, progress_cb=_emit ) is_slave = False slave_identifier = "" master_ip_for_slave = "" if slave_config is None: logger.info("[discovery] %s — could not read config, skipping", candidate_ip) else: for _, master_host, _, _ in rack_hosts: ok, _, _ = parse_slave_check(slave_config, master_host, candidate_ip) if ok: is_slave = True master_ip_for_slave = master_host master_ext_id = next( (ext_id for ext_id, h, _, _ in rack_hosts if h == master_host), master_host, ) slave_identifier = f"{master_ext_id}-slave" break if not is_slave: logger.info("[discovery] %s — not a slave rack", candidate_ip) continue logger.info("[discovery] %s — SLAVE confirmed (name=%s, master=%s)", candidate_ip, slave_identifier, master_ip_for_slave) await _emit("disc_action", {"message": f"Слейв подтверждён: {slave_identifier} ({candidate_ip})"}) slave_rack_id = await _ensure_slave_rack(db, obj, slave_identifier, result) await _upsert_slave_device( db, obj, candidate_ip, slave_identifier, slave_rack_id, master_ip_for_slave, slave_ssh_port, now, result ) all_rack_ips_with_slave = list(all_rack_ips_set) + [candidate_ip] slave_devices = parse_config(slave_config, candidate_ip, all_rack_ips_with_slave, obj.server_ip) logger.info("[discovery] Slave %s: found %d plugin devices", candidate_ip, len(slave_devices)) await _emit("disc_slave_found", { "rack_name": slave_identifier, "ip": candidate_ip, "found": len(slave_devices) }) for found in slave_devices: ping_online, ping_ms = await emit_ping_status( found.ip, _emit, phase="discovered_device" ) await _upsert_plugin_device( db, obj, found, external_id="", rack_id_mapping={}, result=result, slave_rack_id=slave_rack_id, slave_identifier=slave_identifier, ping_online=ping_online, ping_ms=ping_ms if ping_online else None, checked_at=now, ) try: await db.flush() except Exception as exc: result.errors.append(f"Flush error after slave {candidate_ip}: {exc}") logger.error("[discovery] Flush error after slave %s: %s", candidate_ip, exc) except Exception as exc: result.errors.append(f"Slave candidate {candidate_ip}: {exc}") logger.error("[discovery] Error processing slave candidate %s: %s", candidate_ip, exc, exc_info=True) async def _ensure_slave_rack( db: AsyncSession, obj: Object, slave_identifier: str, result: DiscoveryResult ) -> int | None: """Ensure a Rack entry exists for the slave and return its ID.""" try: existing = (await db.execute( select(Rack).where(Rack.object_id == obj.id, Rack.name == slave_identifier) )).scalar_one_or_none() if existing: return existing.id new_rack = Rack( object_id=obj.id, name=slave_identifier, description="slave (auto-discovered via ARP)", ) db.add(new_rack) await db.flush() logger.info("[discovery] Created rack '%s' (id=%d) for slave", slave_identifier, new_rack.id) return new_rack.id except Exception as exc: result.errors.append(f"Error creating rack entry for slave: {exc}") return None async def _upsert_slave_device( db: AsyncSession, obj: Object, candidate_ip: str, slave_identifier: str, slave_rack_id: int | None, master_ip: str, slave_ssh_port: int | None, now: datetime, result: DiscoveryResult, ) -> None: slave_port_override = ( slave_ssh_port if slave_ssh_port and slave_ssh_port != (obj.ssh_port or 22) else None ) try: existing = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == candidate_ip) )).scalar_one_or_none() if existing is None: db.add(Device( object_id=obj.id, ip=candidate_ip, hostname=slave_identifier, category="embedded", role="other", source="auto_ssh", rack_id=slave_rack_id, status="online", last_seen=now, ssh_port_override=slave_port_override, )) logger.info("[discovery] %s (%s) - ADDED: slave rack PC", candidate_ip, slave_identifier) result.created += 1 result.actions.append(DeviceAction( ip=candidate_ip, hostname=slave_identifier, action="ADDED", reason=f"slave rack PC (master={master_ip})", category="embedded", role="other", )) else: changes = [] if existing.status != "online": existing.status = "online" changes.append("status: →online") existing.last_seen = now changes.append("last_seen: updated") if existing.ssh_port_override != slave_port_override: existing.ssh_port_override = slave_port_override changes.append(f"ssh_port_override: ->{slave_port_override or 'default'}") if existing.rack_id is None and slave_rack_id: existing.rack_id = slave_rack_id changes.append(f"rack_id: →{slave_rack_id}") result.updated += 1 result.actions.append(DeviceAction( ip=candidate_ip, hostname=existing.hostname or slave_identifier, action="UPDATED", reason="; ".join(changes), category="embedded", role="other", )) await db.flush() except Exception as exc: result.errors.append(f"Error upserting slave device {candidate_ip}: {exc}") async def _discover_mikrotik( db: AsyncSession, obj: Object, rack_hosts: list[tuple], ssh_port: int, ssh_options: list, result: DiscoveryResult, _emit, ) -> None: target_host = obj.server_ip or rack_hosts[0][1] logger.info("[discovery] Phase 3: MikroTik via SSH_CLIENT on %s", target_host) mikrotik_ip = await discover_mikrotik_multi(target_host, ssh_port, ssh_options, progress_cb=_emit) if not mikrotik_ip: logger.info("[discovery] MikroTik: no IP found via SSH_CLIENT") return logger.info("[discovery] MikroTik candidate: %s — verifying...", mikrotik_ip) await _emit("disc_action", {"message": f"Проверка MikroTik {mikrotik_ip} (WinBox/SSH)..."}) is_mikrotik = await verify_is_mikrotik(mikrotik_ip, ssh_port, ssh_options, progress_cb=_emit) if not is_mikrotik: logger.info("[discovery] %s — NOT a MikroTik (WinBox and SSH banner check failed), skipping", mikrotik_ip) await _emit("disc_action", {"message": f"{mikrotik_ip} — не MikroTik, пропускаем"}) return logger.info("[discovery] MikroTik confirmed at %s", mikrotik_ip) await _emit("disc_infra_found", {"type": "mikrotik", "ip": mikrotik_ip}) try: existing = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == mikrotik_ip) )).scalar_one_or_none() if existing is not None: logger.info("[discovery] %s (mikrotik) - SKIPPED: already exists", mikrotik_ip) result.skipped += 1 result.actions.append(DeviceAction( ip=mikrotik_ip, hostname="mikrotik", action="SKIPPED", reason=f"already exists (source={existing.source}, category={existing.category})", category=existing.category, role=existing.role, )) else: db.add(Device( object_id=obj.id, ip=mikrotik_ip, hostname="mikrotik", category="router", role="other", source="auto_ssh", )) logger.info("[discovery] %s (mikrotik) - ADDED: category=router", mikrotik_ip) result.created += 1 result.actions.append(DeviceAction( ip=mikrotik_ip, hostname="mikrotik", action="ADDED", reason="discovered via SSH_CLIENT", category="router", role="other", )) except Exception as exc: logger.error("[discovery] %s (mikrotik) - ERROR: %s", mikrotik_ip, exc) result.errors.append(f"Error upserting {mikrotik_ip}: {exc}") result.actions.append(DeviceAction( ip=mikrotik_ip, hostname="mikrotik", action="ERROR", reason=str(exc), category="router", role="other", )) try: await db.flush() except Exception as exc: result.errors.append(f"Flush error after mikrotik upsert: {exc}") logger.error("[discovery] Flush error after mikrotik: %s", exc) async def _discover_proxmox( db: AsyncSession, obj: Object, ssh_port: int, ssh_password: str, rack_hosts: list[tuple], result: DiscoveryResult, _emit, ) -> None: target_host = obj.server_ip or rack_hosts[0][1] logger.info("[discovery] Phase 4: Proxmox port 8006 scan on %s", target_host) proxmox_ips = await discover_proxmox_via_port_scan( host=target_host, port=obj.ssh_port or 22, username=obj.ssh_user, password=ssh_password, ) if not proxmox_ips: logger.info("[discovery] Proxmox: no servers found on port 8006") return logger.info("[discovery] Proxmox: found %d server(s): %s", len(proxmox_ips), proxmox_ips) for proxmox_ip in proxmox_ips: await _emit("disc_infra_found", {"type": "proxmox", "ip": proxmox_ip}) for idx, proxmox_ip in enumerate(proxmox_ips, 1): hostname = f"proxmox_{idx}" try: existing = (await db.execute( select(Device).where(Device.object_id == obj.id, Device.ip == proxmox_ip) )).scalar_one_or_none() if existing is not None: logger.info("[discovery] %s (%s) - SKIPPED: already exists", proxmox_ip, hostname) result.skipped += 1 result.actions.append(DeviceAction( ip=proxmox_ip, hostname=hostname, action="SKIPPED", reason=f"already exists (source={existing.source}, category={existing.category})", category=existing.category, role=existing.role, )) else: db.add(Device( object_id=obj.id, ip=proxmox_ip, hostname=hostname, category="vm", role="other", source="auto_ssh", )) logger.info("[discovery] %s (%s) - ADDED: category=vm", proxmox_ip, hostname) result.created += 1 result.actions.append(DeviceAction( ip=proxmox_ip, hostname=hostname, action="ADDED", reason="discovered via port 8006 scan", category="vm", role="other", )) except Exception as exc: logger.error("[discovery] %s (%s) - ERROR: %s", proxmox_ip, hostname, exc) result.errors.append(f"Error upserting {proxmox_ip}: {exc}") result.actions.append(DeviceAction( ip=proxmox_ip, hostname=hostname, action="ERROR", reason=str(exc), category="vm", role="other", )) try: await db.flush() except Exception as exc: result.errors.append(f"Flush error after proxmox upsert: {exc}") logger.error("[discovery] Flush error after proxmox: %s", exc) # ── Summary logging ─────────────────────────────────────────────────────────── def _log_summary(result: DiscoveryResult) -> None: sep = "=" * 100 logger.info(sep) logger.info( "[discovery] SUMMARY: ADDED=%d, UPDATED=%d, SKIPPED=%d, ERRORS=%d", result.created, result.updated, result.skipped, len(result.errors), ) by_action: dict[str, list[DeviceAction]] = {} for action in result.actions: by_action.setdefault(action.action, []).append(action) for label, actions in by_action.items(): log_fn = logger.error if label == "ERROR" else logger.info log_fn("[discovery] ─── %s devices ───", label) for a in actions: log_fn(" %-20s (%-30s) | %-15s | %-10s | %s", a.ip, a.hostname, a.category, a.role, a.reason) logger.info(sep)