From a774b613c7be0fa1e16fe5b083bb01303491d03b Mon Sep 17 00:00:00 2001 From: dv Date: Wed, 29 Apr 2026 12:54:35 +0300 Subject: [PATCH] =?UTF-8?q?=D1=84=D0=B8=D0=BA=D1=81=20=D1=84=D0=B8=D0=BA?= =?UTF-8?q?=D1=81=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/discovery/db_resolver.py | 37 ++++++++++-- .../app/services/discovery/orchestrator.py | 58 +++++++++++++++---- backend/app/services/discovery/ssh_client.py | 20 +++++-- 3 files changed, 95 insertions(+), 20 deletions(-) diff --git a/backend/app/services/discovery/db_resolver.py b/backend/app/services/discovery/db_resolver.py index d7b470d..76e4009 100644 --- a/backend/app/services/discovery/db_resolver.py +++ b/backend/app/services/discovery/db_resolver.py @@ -27,12 +27,9 @@ async def autodetect_db_from_server( """ from configparser import RawConfigParser - try: - config_text = await ssh_run_multi( - server_ip, ssh_port, ssh_options, f"cat {server_config_path}" - ) - except Exception: - return None + config_text = await _read_server_config_with_retry( + server_ip, ssh_port, ssh_options, server_config_path + ) parser = RawConfigParser(strict=False) parser.read_string(config_text) @@ -52,6 +49,34 @@ async def autodetect_db_from_server( return None +async def _read_server_config_with_retry( + server_ip: str, + ssh_port: int, + ssh_options: list, + server_config_path: str, + attempts: int = 2, +) -> str: + """Read server config, retrying once for transient command timeouts.""" + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return await ssh_run_multi( + server_ip, ssh_port, ssh_options, f"cat {server_config_path}", timeout=30 + ) + except TimeoutError as exc: + last_exc = exc + if attempt < attempts: + logger.warning( + "[discovery] Reading server config from %s timed out; retrying (%d/%d)", + server_ip, attempt + 1, attempts, + ) + continue + raise + except Exception: + raise + raise last_exc or RuntimeError("server config read failed") + + async def get_rack_hosts( host: str, port: int, diff --git a/backend/app/services/discovery/orchestrator.py b/backend/app/services/discovery/orchestrator.py index 908b5f1..eaa94f4 100644 --- a/backend/app/services/discovery/orchestrator.py +++ b/backend/app/services/discovery/orchestrator.py @@ -257,9 +257,16 @@ async def _resolve_db_connection( return None elif obj.server_ip: await _emit("disc_action", {"message": f"Чтение конфигурации сервера {obj.server_ip}..."}) - detected = await autodetect_db_from_server( - server_ip=obj.server_ip, ssh_port=ssh_port, ssh_options=ssh_options, - ) + try: + detected = await autodetect_db_from_server( + server_ip=obj.server_ip, ssh_port=ssh_port, ssh_options=ssh_options, + ) + except Exception as exc: + result.errors.append( + f"Could not read server config for auto-detection: {exc}. " + f"Fill in DB fields manually on the object." + ) + return None if not detected or not all([detected.get("host"), detected.get("user"), detected.get("dbname")]): await _emit_db_autodetect_error(obj, ssh_port, ssh_options, result) return None @@ -283,13 +290,25 @@ async def _resolve_db_connection( async def _emit_db_autodetect_error(obj, ssh_port, ssh_options, result: DiscoveryResult) -> None: try: - raw = await ssh_run_multi(obj.server_ip, ssh_port, ssh_options, "cat /etc/caps/config.ini") + raw = await ssh_run_multi( + obj.server_ip, ssh_port, ssh_options, "cat /etc/caps/config.ini", timeout=30 + ) diag_parser = RawConfigParser(strict=False) diag_parser.read_string(raw) found_sections = diag_parser.sections() found_keys = {s: list(diag_parser.options(s)) for s in found_sections} + found_db_uri_sections = [ + s for s in found_sections + if diag_parser.get(s, "db_uri", fallback=None) + ] + db_uri_note = ( + f" db_uri keys were found in sections {found_db_uri_sections}, " + f"but none could be parsed as a PostgreSQL URI." + if found_db_uri_sections else "" + ) result.errors.append( f"db_uri not found in server config. " + f"{db_uri_note}" f"Sections found: {found_sections}. " f"Keys per section: {found_keys}. " f"Fill in DB fields manually on the object." @@ -468,12 +487,31 @@ async def _ssh_fetch_rack_config( Returns (ssh_ok, config_text). """ await _emit("disc_action", {"message": f"SSH → {rack_host} ({rack_name})"}) - try: - config_text = await ssh_run_multi(rack_host, ssh_port, ssh_options, f"cat {config_path}") - return True, config_text - except Exception as exc: - result.errors.append(f"SSH to {rack_host} failed: {exc}") - return False, None + last_exc: Exception | None = None + for attempt in range(1, 3): + try: + config_text = await ssh_run_multi( + rack_host, ssh_port, ssh_options, f"cat {config_path}", timeout=30 + ) + return True, config_text + except TimeoutError as exc: + last_exc = exc + if attempt < 2: + logger.warning( + "[discovery] Reading rack config from %s timed out; retrying (%d/2)", + rack_host, attempt + 1, + ) + await _emit("disc_action", { + "message": f"Повторное чтение конфигурации {rack_host} ({rack_name})..." + }) + continue + break + except Exception as exc: + last_exc = exc + break + + result.errors.append(f"SSH to {rack_host} failed: {last_exc}") + return False, None async def _upsert_rack_device( diff --git a/backend/app/services/discovery/ssh_client.py b/backend/app/services/discovery/ssh_client.py index d55aaef..1511551 100644 --- a/backend/app/services/discovery/ssh_client.py +++ b/backend/app/services/discovery/ssh_client.py @@ -141,10 +141,22 @@ async def fetch_slave_config( host: str, port: int, options: list, config_path: str, ) -> str | None: """Read a remote config file from a slave candidate. Returns None on failure.""" - try: - return await ssh_run_multi(host, port, options, f"cat {config_path}", timeout=15) - except Exception: - return None + for attempt in range(1, 3): + try: + return await ssh_run_multi(host, port, options, f"cat {config_path}", timeout=30) + except TimeoutError as exc: + if attempt < 2: + logger.warning( + "[discovery] Reading slave config from %s timed out; retrying (%d/2)", + host, attempt + 1, + ) + continue + logger.warning("[discovery] Reading slave config from %s failed: %s", host, exc) + return None + except Exception as exc: + logger.warning("[discovery] Reading slave config from %s failed: %s", host, exc) + return None + return None async def discover_proxmox_via_port_scan(