165 lines
5 KiB
Python
165 lines
5 KiB
Python
"""
|
|
Google Sheets sync service.
|
|
|
|
Reads device rows from a Google Spreadsheet and upserts them into the devices table.
|
|
Column mapping (case-insensitive): ip, hostname, category, role, model, mac, notes.
|
|
|
|
Prerequisites:
|
|
- Set GOOGLE_SERVICE_ACCOUNT_JSON env var with a service account JSON key.
|
|
- Share the target spreadsheet with the service account email (read-only is enough).
|
|
|
|
gspread is synchronous; calls are wrapped in run_in_executor to avoid blocking.
|
|
"""
|
|
|
|
import asyncio
|
|
import functools
|
|
import json
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.device import Device
|
|
|
|
try:
|
|
import gspread
|
|
from google.oauth2.service_account import Credentials as ServiceAccountCredentials
|
|
|
|
_GSPREAD_AVAILABLE = True
|
|
except ImportError:
|
|
_GSPREAD_AVAILABLE = False
|
|
|
|
_SCOPES = [
|
|
"https://spreadsheets.google.com/feeds",
|
|
"https://www.googleapis.com/auth/drive.readonly",
|
|
]
|
|
|
|
# Supported column names (lowercase) → Device field
|
|
_COLUMN_MAP = {
|
|
"ip": "ip",
|
|
"hostname": "hostname",
|
|
"category": "category",
|
|
"role": "role",
|
|
"model": "model",
|
|
"mac": "mac",
|
|
"notes": "notes",
|
|
}
|
|
|
|
_VALID_CATEGORIES = {"raspberry", "mikrotik", "camera", "embedded", "server", "other"}
|
|
|
|
|
|
@dataclass
|
|
class SheetsSyncResult:
|
|
created: int = 0
|
|
updated: int = 0
|
|
skipped: int = 0
|
|
errors: list[str] = field(default_factory=list)
|
|
|
|
|
|
def _get_gspread_client():
|
|
if not _GSPREAD_AVAILABLE:
|
|
raise ValueError(
|
|
"gspread is not installed. Add 'gspread' to project dependencies."
|
|
)
|
|
|
|
creds_raw = os.environ.get("GOOGLE_SERVICE_ACCOUNT_JSON")
|
|
if not creds_raw:
|
|
raise ValueError(
|
|
"GOOGLE_SERVICE_ACCOUNT_JSON environment variable is not set. "
|
|
"Provide a service account JSON key to enable Google Sheets sync."
|
|
)
|
|
|
|
creds_data = json.loads(creds_raw)
|
|
creds = ServiceAccountCredentials.from_service_account_info(creds_data, scopes=_SCOPES)
|
|
return gspread.authorize(creds)
|
|
|
|
|
|
def _fetch_rows_sync(spreadsheet_id: str, sheet_name: Optional[str]) -> list[dict]:
|
|
"""Synchronous fetch — runs in executor."""
|
|
client = _get_gspread_client()
|
|
sh = client.open_by_key(spreadsheet_id)
|
|
ws = sh.worksheet(sheet_name) if sheet_name else sh.sheet1
|
|
return ws.get_all_records() # list of dicts keyed by header row
|
|
|
|
|
|
async def fetch_rows(spreadsheet_id: str, sheet_name: Optional[str]) -> list[dict]:
|
|
loop = asyncio.get_event_loop()
|
|
return await loop.run_in_executor(
|
|
None,
|
|
functools.partial(_fetch_rows_sync, spreadsheet_id, sheet_name),
|
|
)
|
|
|
|
|
|
def _normalize_row(raw: dict) -> dict:
|
|
"""Lowercase keys, strip values."""
|
|
return {k.lower().strip(): str(v).strip() for k, v in raw.items() if v != ""}
|
|
|
|
|
|
async def sync_from_sheets(
|
|
db: AsyncSession,
|
|
object_id: int,
|
|
spreadsheet_id: str,
|
|
sheet_name: Optional[str] = None,
|
|
) -> SheetsSyncResult:
|
|
result = SheetsSyncResult()
|
|
|
|
# 1. Fetch rows from Google Sheets (blocking I/O in executor)
|
|
try:
|
|
raw_rows = await fetch_rows(spreadsheet_id, sheet_name)
|
|
except Exception as exc:
|
|
result.errors.append(f"Failed to read spreadsheet: {exc}")
|
|
return result
|
|
|
|
# 2. Load existing devices for this object (keyed by ip)
|
|
existing_result = await db.execute(
|
|
select(Device).where(Device.object_id == object_id)
|
|
)
|
|
existing: dict[str, Device] = {d.ip: d for d in existing_result.scalars().all()}
|
|
|
|
for i, raw in enumerate(raw_rows, start=2): # row 2 = first data row (after header)
|
|
row = _normalize_row(raw)
|
|
ip = row.get("ip", "").strip()
|
|
|
|
if not ip:
|
|
result.errors.append(f"Row {i}: missing 'ip' column — skipped")
|
|
result.skipped += 1
|
|
continue
|
|
|
|
category = row.get("category", "other")
|
|
if category not in _VALID_CATEGORIES:
|
|
category = "other"
|
|
|
|
if ip in existing:
|
|
dev = existing[ip]
|
|
changed = False
|
|
for col, field_name in _COLUMN_MAP.items():
|
|
if col in row and col != "ip":
|
|
new_val = row[col] or None
|
|
if getattr(dev, field_name) != new_val:
|
|
setattr(dev, field_name, new_val)
|
|
changed = True
|
|
dev.source = "sheets_sync"
|
|
if changed:
|
|
result.updated += 1
|
|
else:
|
|
result.skipped += 1
|
|
else:
|
|
dev = Device(
|
|
object_id=object_id,
|
|
ip=ip,
|
|
hostname=row.get("hostname") or None,
|
|
category=category,
|
|
role=row.get("role", "other"),
|
|
model=row.get("model") or None,
|
|
mac=row.get("mac") or None,
|
|
notes=row.get("notes") or None,
|
|
source="sheets_sync",
|
|
status="unknown",
|
|
)
|
|
db.add(dev)
|
|
existing[ip] = dev
|
|
result.created += 1
|
|
|
|
return result
|