458 lines
16 KiB
Python
458 lines
16 KiB
Python
"""
|
|
Low-level SSH operations for device discovery.
|
|
No SQLAlchemy, no business logic — pure network I/O.
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import re
|
|
import socket
|
|
import time
|
|
from collections.abc import Awaitable, Callable
|
|
|
|
import asyncssh
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ProgressCallback = Callable[[str, dict], Awaitable[None]]
|
|
|
|
PING_TIMEOUT = 3
|
|
SSH_CONNECT_TIMEOUT = 15
|
|
|
|
DISCOVERY_SSH_KWARGS: dict = {
|
|
"kex_algs": ["curve25519-sha256"],
|
|
"encryption_algs": ["aes128-ctr"],
|
|
"mac_algs": ["hmac-sha2-256"],
|
|
}
|
|
|
|
|
|
async def _connect_with_sock_compat(**kwargs) -> asyncssh.SSHClientConnection:
|
|
"""Connect with IPQoS disabled and conservative SSH algorithms.
|
|
|
|
AsyncSSH doesn't expose OpenSSH's `IPQoS=none` option directly, so discovery
|
|
opens the TCP socket itself, clears IP_TOS, and passes the connected socket
|
|
to AsyncSSH.
|
|
"""
|
|
host = kwargs.pop("host")
|
|
port = kwargs.pop("port", 22)
|
|
connect_timeout = kwargs.pop("connect_timeout", 10)
|
|
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
sock.setblocking(False)
|
|
try:
|
|
try:
|
|
sock.setsockopt(socket.IPPROTO_IP, socket.IP_TOS, 0)
|
|
except OSError:
|
|
logger.debug("[discovery] Could not clear IP_TOS for %s:%s", host, port)
|
|
|
|
loop = asyncio.get_running_loop()
|
|
await asyncio.wait_for(loop.sock_connect(sock, (host, port)), timeout=connect_timeout)
|
|
return await asyncssh.connect(
|
|
host=host,
|
|
port=port,
|
|
sock=sock,
|
|
connect_timeout=connect_timeout,
|
|
**DISCOVERY_SSH_KWARGS,
|
|
**kwargs,
|
|
)
|
|
except BaseException:
|
|
sock.close()
|
|
raise
|
|
|
|
|
|
# ── Single connection helpers (used internally and for legacy paths) ──────────
|
|
|
|
async def read_remote_file(
|
|
host: str,
|
|
port: int,
|
|
username: str,
|
|
password: str,
|
|
remote_path: str,
|
|
) -> str:
|
|
"""SSH into host and return the contents of remote_path."""
|
|
if not await is_port_open(host, port, timeout=3.0):
|
|
raise ConnectionRefusedError(f"{host}:{port} — порт недоступен")
|
|
async with await _connect_with_sock_compat(
|
|
host=host, port=port, username=username, password=password,
|
|
known_hosts=None, connect_timeout=10,
|
|
) as conn:
|
|
result = await conn.run(f"cat {remote_path}", check=True)
|
|
return result.stdout
|
|
|
|
|
|
async def get_ip_neighbors(
|
|
host: str, port: int, username: str, password: str,
|
|
) -> list[str]:
|
|
"""Run `ip neigh show` and return REACHABLE/DELAY neighbour IPs."""
|
|
try:
|
|
if not await is_port_open(host, port, timeout=3.0):
|
|
logger.info("[discovery] %s:%d — port closed, skipping ip neigh", host, port)
|
|
return []
|
|
async with await _connect_with_sock_compat(
|
|
host=host, port=port, username=username, password=password,
|
|
known_hosts=None, connect_timeout=10,
|
|
) as conn:
|
|
result = await conn.run("ip neigh show", check=False)
|
|
return _parse_neigh_output(result.stdout)
|
|
except Exception as exc:
|
|
logger.warning("[discovery] ip neigh failed on %s: %s", host, exc)
|
|
return []
|
|
|
|
|
|
# ── TCP port probe ────────────────────────────────────────────────────────────
|
|
|
|
async def is_port_open(host: str, port: int, timeout: float = 3.0) -> bool:
|
|
"""Quick TCP connect check. Returns True if port is reachable, False otherwise."""
|
|
try:
|
|
_, writer = await asyncio.wait_for(
|
|
asyncio.open_connection(host, port), timeout=timeout
|
|
)
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
async def ping_host(host: str, timeout: int = PING_TIMEOUT) -> tuple[bool, int]:
|
|
"""Ping a host once. Returns (success, latency_ms)."""
|
|
start = time.monotonic()
|
|
try:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
"ping", "-c", "1", "-W", str(timeout), host,
|
|
stdout=asyncio.subprocess.DEVNULL,
|
|
stderr=asyncio.subprocess.DEVNULL,
|
|
)
|
|
await asyncio.wait_for(proc.wait(), timeout=timeout + 2)
|
|
ms = int((time.monotonic() - start) * 1000)
|
|
return proc.returncode == 0, ms
|
|
except Exception:
|
|
return False, 0
|
|
|
|
|
|
async def emit_ping_status(
|
|
host: str,
|
|
progress_cb: ProgressCallback | None = None,
|
|
*,
|
|
phase: str = "ssh",
|
|
) -> tuple[bool, int]:
|
|
online, ping_ms = await ping_host(host)
|
|
if progress_cb is not None:
|
|
await progress_cb("disc_device_ping", {
|
|
"ip": host,
|
|
"status": "online" if online else "offline",
|
|
"ping_ms": ping_ms if online else None,
|
|
"phase": phase,
|
|
})
|
|
logger.info(
|
|
"[discovery] ping %s before %s: %s%s",
|
|
host,
|
|
phase,
|
|
"online" if online else "offline",
|
|
f" ({ping_ms} ms)" if online else "",
|
|
)
|
|
return online, ping_ms
|
|
|
|
|
|
# ── Multi-auth helpers ────────────────────────────────────────────────────────
|
|
|
|
async def ssh_run_multi(
|
|
host: str,
|
|
port: int,
|
|
options: list,
|
|
command: str,
|
|
timeout: int = 15,
|
|
progress_cb: ProgressCallback | None = None,
|
|
) -> str:
|
|
"""Try each SSH auth option in order; return stdout on first success or raise."""
|
|
await emit_ping_status(host, progress_cb)
|
|
|
|
# Fast pre-check: skip SSH entirely if port is not open
|
|
if not await is_port_open(host, port, timeout=3.0):
|
|
raise ConnectionRefusedError(f"{host}:{port} — порт недоступен (TCP check failed)")
|
|
|
|
last_exc: Exception = Exception("No auth options provided")
|
|
for i, opt in enumerate(options):
|
|
_log_attempt(i, len(options), host, opt)
|
|
try:
|
|
connect_kwargs = _build_connect_kwargs(host, port, opt)
|
|
conn = await asyncio.wait_for(
|
|
_connect_with_sock_compat(**connect_kwargs, connect_timeout=SSH_CONNECT_TIMEOUT),
|
|
timeout=SSH_CONNECT_TIMEOUT,
|
|
)
|
|
except Exception as exc:
|
|
last_exc = exc
|
|
logger.info(
|
|
"[discovery] SSH auth failed for %s with %s(%s): %s",
|
|
host, opt.type, opt.username, exc,
|
|
)
|
|
continue
|
|
|
|
logger.info(
|
|
"[discovery] SSH auth succeeded for %s with %s(%s)",
|
|
host, opt.type, opt.username,
|
|
)
|
|
try:
|
|
async with conn:
|
|
result = await asyncio.wait_for(
|
|
conn.run(command, check=True), timeout=timeout
|
|
)
|
|
return result.stdout
|
|
except asyncio.TimeoutError as exc:
|
|
err = TimeoutError(f"SSH command timed out after {timeout}s on {host}: {command}")
|
|
logger.warning("[discovery] %s", err)
|
|
raise err from exc
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"[discovery] SSH command failed on %s after auth with %s(%s): %s",
|
|
host, opt.type, opt.username, exc,
|
|
)
|
|
raise
|
|
raise last_exc
|
|
|
|
|
|
async def get_ip_neighbors_multi(
|
|
host: str,
|
|
port: int,
|
|
options: list,
|
|
progress_cb: ProgressCallback | None = None,
|
|
) -> list[str]:
|
|
"""Multi-auth version of get_ip_neighbors."""
|
|
try:
|
|
stdout = await ssh_run_multi(
|
|
host, port, options, "ip neigh show", timeout=15, progress_cb=progress_cb
|
|
)
|
|
return _parse_neigh_output(stdout)
|
|
except Exception as exc:
|
|
logger.warning("[discovery] ip neigh failed on %s: %s", host, exc)
|
|
return []
|
|
|
|
|
|
async def discover_mikrotik_multi(
|
|
host: str,
|
|
port: int,
|
|
options: list,
|
|
progress_cb: ProgressCallback | None = None,
|
|
) -> str | None:
|
|
"""Discover MikroTik IP via SSH_CLIENT environment variable (multi-auth)."""
|
|
try:
|
|
stdout = await ssh_run_multi(
|
|
host, port, options, "echo $SSH_CLIENT", timeout=15, progress_cb=progress_cb
|
|
)
|
|
parts = stdout.strip().split()
|
|
return parts[0] if parts else None
|
|
except Exception as exc:
|
|
logger.warning("[discovery] Failed to discover MikroTik via SSH_CLIENT: %s", exc)
|
|
return None
|
|
|
|
|
|
async def fetch_slave_config(
|
|
host: str,
|
|
ports: list[int],
|
|
options: list,
|
|
config_path: str,
|
|
progress_cb: ProgressCallback | None = None,
|
|
) -> tuple[str | None, int | None]:
|
|
"""Read a remote config file from a slave candidate. Returns (config, port)."""
|
|
for port in ports:
|
|
for attempt in range(1, 3):
|
|
try:
|
|
config_text = await ssh_run_multi(
|
|
host, port, options, f"cat {config_path}", timeout=30, progress_cb=progress_cb
|
|
)
|
|
return config_text, port
|
|
except TimeoutError as exc:
|
|
if attempt < 2:
|
|
logger.warning(
|
|
"[discovery] Reading slave config from %s:%d timed out; retrying (%d/2)",
|
|
host, port, attempt + 1,
|
|
)
|
|
continue
|
|
logger.warning("[discovery] Reading slave config from %s:%d failed: %s", host, port, exc)
|
|
break
|
|
except Exception as exc:
|
|
logger.warning("[discovery] Reading slave config from %s:%d failed: %s", host, port, exc)
|
|
break
|
|
return None, None
|
|
|
|
|
|
async def discover_proxmox_via_port_scan(
|
|
host: str, port: int, username: str, password: str,
|
|
) -> list[str]:
|
|
"""Discover Proxmox servers on the local subnet by scanning port 8006.
|
|
|
|
Note: uses single-auth (password) because it's called from the legacy path.
|
|
Migrate to multi-auth when needed.
|
|
"""
|
|
try:
|
|
async with await _connect_with_sock_compat(
|
|
host=host, port=port, username=username, password=password,
|
|
known_hosts=None, connect_timeout=10,
|
|
) as conn:
|
|
scan_script = (
|
|
"subnet=$(ip -o -f inet addr show | awk '!/ lo |docker|br-|wg/ {print $4; exit}' | "
|
|
"cut -d/ -f1 | awk -F. '{print $1\".\"$2\".\"$3}') && "
|
|
"(set +m; for ip in $subnet.{1..254}; do "
|
|
"timeout 0.3 bash -c \"echo > /dev/tcp/$ip/8006\" 2>/dev/null && echo \"$ip\" & "
|
|
"done; wait)"
|
|
)
|
|
result = await conn.run(scan_script, check=False)
|
|
return [ln.strip() for ln in result.stdout.strip().split("\n") if ln.strip()]
|
|
except Exception as exc:
|
|
logger.warning("[discovery] Proxmox port scan failed on %s: %s", host, exc)
|
|
return []
|
|
|
|
|
|
# ── Tunnel ────────────────────────────────────────────────────────────────────
|
|
|
|
async def open_ssh_tunnel(
|
|
tunnel_host: str,
|
|
tunnel_port: int,
|
|
ssh_options: list,
|
|
target_host: str,
|
|
target_port: int,
|
|
progress_cb: ProgressCallback | None = None,
|
|
):
|
|
"""Open an asyncssh connection, trying each option in order.
|
|
|
|
Returns an open asyncssh connection on first success, or raises.
|
|
The caller is responsible for closing the connection.
|
|
"""
|
|
last_exc: Exception = Exception("No SSH options provided")
|
|
await emit_ping_status(tunnel_host, progress_cb, phase="ssh_tunnel")
|
|
for opt in ssh_options:
|
|
try:
|
|
connect_kwargs = _build_connect_kwargs(tunnel_host, tunnel_port, opt)
|
|
if opt.type == "key":
|
|
logger.info(
|
|
"[discovery] Tunnel attempt for %s — key: user=%s key_path=%s",
|
|
tunnel_host, opt.username, opt.key_path,
|
|
)
|
|
else:
|
|
masked = _mask(opt.password)
|
|
logger.info(
|
|
"[discovery] Tunnel attempt for %s — password: user=%s password=%s",
|
|
tunnel_host, opt.username, masked,
|
|
)
|
|
conn = await asyncio.wait_for(
|
|
_connect_with_sock_compat(**connect_kwargs),
|
|
timeout=SSH_CONNECT_TIMEOUT,
|
|
)
|
|
logger.info(
|
|
"[discovery] Tunnel auth succeeded for %s with %s(%s)",
|
|
tunnel_host, opt.type, opt.username,
|
|
)
|
|
return conn
|
|
except Exception as exc:
|
|
last_exc = exc
|
|
logger.info(
|
|
"[discovery] Tunnel auth failed for %s with %s(%s): %s",
|
|
tunnel_host, opt.type, opt.username, exc,
|
|
)
|
|
raise last_exc
|
|
|
|
|
|
# ── MikroTik verification ─────────────────────────────────────────────────────
|
|
|
|
async def verify_is_mikrotik(
|
|
host: str,
|
|
ssh_port: int = 22,
|
|
ssh_options: list | None = None,
|
|
timeout: float = 3.0,
|
|
progress_cb: ProgressCallback | None = None,
|
|
) -> bool:
|
|
"""Check whether host is actually a MikroTik RouterOS device.
|
|
|
|
Two independent probes are tried (first positive result wins):
|
|
1. TCP connect to port 8291 (WinBox) — exclusively used by MikroTik.
|
|
2. SSH banner check — RouterOS advertises "ROSSSH" as its SSH version.
|
|
|
|
Returns True if either probe confirms MikroTik, False otherwise.
|
|
"""
|
|
# Probe 1: WinBox port 8291 (MikroTik-exclusive)
|
|
try:
|
|
reader, writer = await asyncio.wait_for(
|
|
asyncio.open_connection(host, 8291), timeout=timeout
|
|
)
|
|
writer.close()
|
|
try:
|
|
await writer.wait_closed()
|
|
except Exception:
|
|
pass
|
|
logger.info("[discovery] %s — MikroTik confirmed via WinBox port 8291", host)
|
|
return True
|
|
except Exception:
|
|
pass
|
|
|
|
# Probe 2: SSH banner
|
|
if ssh_options:
|
|
await emit_ping_status(host, progress_cb, phase="mikrotik_verify")
|
|
for opt in ssh_options:
|
|
try:
|
|
kwargs = _build_connect_kwargs(host, ssh_port, opt)
|
|
kwargs["connect_timeout"] = int(timeout)
|
|
async with await _connect_with_sock_compat(**kwargs) as conn:
|
|
banner = (conn.get_extra_info("server_version") or "").lower()
|
|
if "rosssh" in banner or "mikrotik" in banner:
|
|
logger.info(
|
|
"[discovery] %s — MikroTik confirmed via SSH banner: %s", host, banner
|
|
)
|
|
return True
|
|
logger.info(
|
|
"[discovery] %s — SSH connected but banner does not match MikroTik: %s",
|
|
host, banner,
|
|
)
|
|
return False # Connected successfully — definitely not MikroTik
|
|
except Exception:
|
|
continue
|
|
|
|
logger.info("[discovery] %s — could not verify MikroTik (no probe succeeded)", host)
|
|
return False
|
|
|
|
|
|
# ── Internal helpers ──────────────────────────────────────────────────────────
|
|
|
|
def _parse_neigh_output(stdout: str) -> list[str]:
|
|
ips = []
|
|
for line in stdout.strip().splitlines():
|
|
parts = line.split()
|
|
if not parts:
|
|
continue
|
|
ip = parts[0]
|
|
state = parts[-1]
|
|
if re.match(r"^\d{1,3}(\.\d{1,3}){3}$", ip) and state in ("REACHABLE", "DELAY"):
|
|
ips.append(ip)
|
|
return ips
|
|
|
|
|
|
def _build_connect_kwargs(host: str, port: int, opt) -> dict:
|
|
kwargs: dict = dict(
|
|
host=host, port=port, username=opt.username,
|
|
known_hosts=None, connect_timeout=10,
|
|
)
|
|
if opt.type == "key":
|
|
kwargs["client_keys"] = [opt.key_path]
|
|
kwargs["passphrase"] = None
|
|
else:
|
|
kwargs["password"] = opt.password
|
|
return kwargs
|
|
|
|
|
|
def _mask(password: str) -> str:
|
|
return (password[:1] + "***" + password[-1:]) if len(password) > 2 else "***"
|
|
|
|
|
|
def _log_attempt(idx: int, total: int, host: str, opt) -> None:
|
|
if opt.type == "key":
|
|
logger.info(
|
|
"[discovery] SSH attempt %d/%d for %s — key: user=%s key_path=%s",
|
|
idx + 1, total, host, opt.username, opt.key_path,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"[discovery] SSH attempt %d/%d for %s — password: user=%s password=%s",
|
|
idx + 1, total, host, opt.username, _mask(opt.password),
|
|
)
|