vibe 1

Dockerfile

@@ -18,6 +18,7 @@ RUN npm install --omit=dev --no-audit --no-fund \
   && npm cache clean --force
 
 COPY server.js ./
+COPY refresh-service.js ./
 COPY public/ ./public/
 
 # Папка для базы данных

public/js/auth.js

@@ -19,6 +19,7 @@ document.addEventListener('keydown',e=>{
 function doLogout() {
   localStorage.removeItem('intradesk_token');
   authToken=null; currentUser=null;
+  reloadInFlight=false;
   if(statusPollTimer)      clearInterval(statusPollTimer);
   if(clientCountdownTimer) clearInterval(clientCountdownTimer);
   stopSSE();
@@ -51,7 +52,13 @@ async function init() {
     loadUsersTable();
   }
   const status=await apiGet('/api/status');
-  if(status){serverRefreshInterval=status.auto_refresh_interval||300;lastKnownUpdatedAt=status.last_updated;}
+  if(status){
+    serverRefreshInterval=status.auto_refresh_interval||300;
+    lastKnownUpdatedAt=status.last_updated;
+    if(status.cache_version!==undefined&&status.cache_version!==null){
+      lastKnownCacheVersion=Number(status.cache_version)||0;
+    }
+  }
   startSSE();
   startStatusPolling();
 }
@@ -76,6 +83,9 @@ async function loadTasksFromServer(silent=false) {
     return {task,lastComment:lc};
   });
   lastKnownUpdatedAt=data.updated_at;
+  if(data.cache_version!==undefined&&data.cache_version!==null){
+    lastKnownCacheVersion=Number(data.cache_version)||lastKnownCacheVersion;
+  }
   const el=id=>document.getElementById(id);
   if(el('total-loaded')) el('total-loaded').textContent=allData.length;
   if(el('total-count'))  el('total-count').textContent=allData.length;
@@ -146,18 +156,31 @@ async function loadAllTasks() {
 }
 
 async function reloadData() {
-  await Promise.all([loadMarks(),loadNotes()]);
-  await loadTasksFromServer(true);
+  if(reloadInFlight) return;
+  reloadInFlight=true;
+  try {
+    await Promise.all([loadMarks(),loadNotes()]);
+    await loadTasksFromServer(true);
+  } finally {
+    reloadInFlight=false;
+  }
 }
 
 function startStatusPolling() {
   if(statusPollTimer) clearInterval(statusPollTimer);
   statusPollTimer=setInterval(async()=>{
     const s=await apiGet('/api/status');
+    if(!s) return;
+    const serverVersion=Number(s.cache_version||0);
+    if(serverVersion&&serverVersion!==lastKnownCacheVersion){
+      lastKnownCacheVersion=serverVersion;
+      if(s.last_updated) lastKnownUpdatedAt=s.last_updated;
+      await reloadData();
+      return;
+    }
     if(s?.last_updated && s.last_updated!==lastKnownUpdatedAt) {
       lastKnownUpdatedAt=s.last_updated;
-      await Promise.all([loadMarks(),loadNotes()]);
-      await loadTasksFromServer(true);
+      await reloadData();
     }
   },30000);
 }

public/js/diff.js

@@ -136,6 +136,9 @@ function applyTasksDiff(newRawTasks, silent = false) {
 
     // Обновляем мета-данные (те же строки, что были в оригинале)
     lastKnownUpdatedAt = data.updated_at;
+    if (data.cache_version !== undefined && data.cache_version !== null) {
+      lastKnownCacheVersion = Number(data.cache_version) || lastKnownCacheVersion;
+    }
     const el = id => document.getElementById(id);
     if (el('total-loaded')) el('total-loaded').textContent = allData.length;
     if (el('total-count'))  el('total-count').textContent  = allData.length;

public/js/sse.js

@@ -86,6 +86,9 @@ function startSSE() {
 
   sseSource.addEventListener('tasks_updated',e=>{
     const d=JSON.parse(e.data);
+    const incomingVersion=Number(d.cache_version||0);
+    if(incomingVersion&&incomingVersion===lastKnownCacheVersion) return;
+    if(incomingVersion) lastKnownCacheVersion=incomingVersion;
     showToast(`🔄 Обновление: ${d.count} заявок`,'info',4000);
     reloadData();
   });

public/js/state.js

@@ -25,6 +25,8 @@ let mentionsData  = [];
 let mentionUnread = 0;
 let knownUsers    = [];
 let lastKnownUpdatedAt = null;
+let lastKnownCacheVersion = 0;
+let reloadInFlight = false;
 let statusPollTimer = null, clientCountdownTimer = null, clientCountdownSec = 0;
 let serverRefreshInterval = 300;
 let commentCounts = {};

refresh-service.js

@@ -0,0 +1,576 @@
+'use strict';
+
+const DEFAULT_REFRESH_MODE = 'incremental_scan';
+const DEFAULT_SCAN_PAGE_SIZE = 50;
+const DEFAULT_DAILY_FULL_REFRESH_HOUR = 4;
+const SCHEDULER_TICK_MS = 60 * 1000;
+const MAX_SCAN_PAGES = 500;
+
+const CHANGE_EVENT_TYPES = new Set([
+  'created',
+  'updated',
+  'comment',
+  'status',
+  'executor_changed',
+]);
+
+const REMOVE_EVENT_TYPES = new Set([
+  'archived',
+  'out_of_scope',
+  'deleted',
+]);
+
+function safeIsoNow() {
+  return new Date().toISOString();
+}
+
+function toInt(value, fallback) {
+  const n = parseInt(value, 10);
+  return Number.isFinite(n) ? n : fallback;
+}
+
+function normalizeRefreshMode(mode) {
+  return mode === 'webhook_daily' ? 'webhook_daily' : DEFAULT_REFRESH_MODE;
+}
+
+function normalizePageSize(value) {
+  const n = toInt(value, DEFAULT_SCAN_PAGE_SIZE);
+  return Math.max(10, Math.min(500, n));
+}
+
+function normalizeDailyHour(value) {
+  const n = toInt(value, DEFAULT_DAILY_FULL_REFRESH_HOUR);
+  return Math.max(0, Math.min(23, n));
+}
+
+function sortByUpdatedDesc(tasks) {
+  return [...tasks].sort((a, b) => {
+    const au = a?.updatedat || '';
+    const bu = b?.updatedat || '';
+    if (bu === au) return 0;
+    return bu > au ? 1 : -1;
+  });
+}
+
+function extractTaskId(task) {
+  return String(task?.id ?? '');
+}
+
+function extractTaskNumber(task) {
+  return String(task?.tasknumber ?? task?.id ?? '');
+}
+
+function extractUpdatedAt(task) {
+  return String(task?.updatedat ?? '');
+}
+
+function hasChangesByIdAndUpdated(oldTasks, newTasks) {
+  if (oldTasks.length !== newTasks.length) return true;
+  const oldMap = new Map(oldTasks.map((t) => [extractTaskId(t), extractUpdatedAt(t)]));
+  for (const task of newTasks) {
+    const id = extractTaskId(task);
+    if (!oldMap.has(id)) return true;
+    if (oldMap.get(id) !== extractUpdatedAt(task)) return true;
+  }
+  return false;
+}
+
+function createRefreshService({ db, fetch, getSettings, sseBroadcast, logger = console }) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS sync_state (
+      id                      INTEGER PRIMARY KEY CHECK (id = 1),
+      cache_version           INTEGER NOT NULL DEFAULT 0,
+      refresh_in_progress     INTEGER NOT NULL DEFAULT 0,
+      last_success_at         TEXT,
+      last_error              TEXT,
+      last_full_reconcile_at  TEXT
+    );
+
+    INSERT OR IGNORE INTO sync_state(id, cache_version, refresh_in_progress)
+    VALUES(1, 0, 0);
+
+    CREATE TABLE IF NOT EXISTS tasks_index (
+      task_id     TEXT PRIMARY KEY,
+      task_number TEXT NOT NULL,
+      updated_at  TEXT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_tasks_index_number ON tasks_index(task_number);
+  `);
+
+  const stmts = {
+    getCacheRow: db.prepare('SELECT data, updated_at FROM tasks_cache WHERE id=1'),
+    upsertCache: db.prepare(`
+      INSERT INTO tasks_cache(id,data,updated_at) VALUES(1,?,?)
+      ON CONFLICT(id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at
+    `),
+    getSyncState: db.prepare(`
+      SELECT cache_version, refresh_in_progress, last_success_at, last_error, last_full_reconcile_at
+      FROM sync_state WHERE id=1
+    `),
+    setInProgress: db.prepare('UPDATE sync_state SET refresh_in_progress=? WHERE id=1'),
+    markError: db.prepare(`
+      UPDATE sync_state
+      SET refresh_in_progress=0, last_error=?
+      WHERE id=1
+    `),
+    markSuccessNoChange: db.prepare(`
+      UPDATE sync_state
+      SET refresh_in_progress=0, last_success_at=?, last_error=?
+      WHERE id=1
+    `),
+    markSuccessNoChangeFull: db.prepare(`
+      UPDATE sync_state
+      SET refresh_in_progress=0, last_success_at=?, last_error=?, last_full_reconcile_at=?
+      WHERE id=1
+    `),
+    writeStateAfterCache: db.prepare(`
+      UPDATE sync_state
+      SET cache_version=?, refresh_in_progress=0, last_success_at=?, last_error=?
+      WHERE id=1
+    `),
+    writeStateAfterFull: db.prepare(`
+      UPDATE sync_state
+      SET cache_version=?, refresh_in_progress=0, last_success_at=?, last_error=?, last_full_reconcile_at=?
+      WHERE id=1
+    `),
+    clearIndex: db.prepare('DELETE FROM tasks_index'),
+    insertIndex: db.prepare('INSERT INTO tasks_index(task_id, task_number, updated_at) VALUES(?,?,?)'),
+  };
+
+  function getSyncState() {
+    const row = stmts.getSyncState.get();
+    if (row) return row;
+    return {
+      cache_version: 0,
+      refresh_in_progress: 0,
+      last_success_at: null,
+      last_error: null,
+      last_full_reconcile_at: null,
+    };
+  }
+
+  function readCachedTasks() {
+    try {
+      const row = stmts.getCacheRow.get();
+      if (!row) return { tasks: [], updated_at: null };
+      return {
+        tasks: JSON.parse(row.data || '[]'),
+        updated_at: row.updated_at || null,
+      };
+    } catch {
+      return { tasks: [], updated_at: null };
+    }
+  }
+
+  function normalizeSettings() {
+    const s = getSettings();
+    return {
+      ...s,
+      refresh_mode: normalizeRefreshMode(s.refresh_mode),
+      scan_page_size: normalizePageSize(s.scan_page_size),
+      daily_full_refresh_hour: normalizeDailyHour(s.daily_full_refresh_hour),
+    };
+  }
+
+  function writeCacheAndIndexes(tasks, { fullReconcile = false } = {}) {
+    const now = safeIsoNow();
+    const state = getSyncState();
+    const nextVersion = Number(state.cache_version || 0) + 1;
+    const sorted = sortByUpdatedDesc(tasks);
+
+    const tx = db.transaction(() => {
+      stmts.upsertCache.run(JSON.stringify(sorted), now);
+      stmts.clearIndex.run();
+      for (const task of sorted) {
+        const taskId = extractTaskId(task);
+        if (!taskId) continue;
+        stmts.insertIndex.run(taskId, extractTaskNumber(task), extractUpdatedAt(task) || now);
+      }
+      if (fullReconcile) {
+        stmts.writeStateAfterFull.run(nextVersion, now, null, now);
+      } else {
+        stmts.writeStateAfterCache.run(nextVersion, now, null);
+      }
+    });
+    tx();
+
+    sseBroadcast('tasks_updated', {
+      updated_at: now,
+      count: sorted.length,
+      cache_version: nextVersion,
+    });
+
+    return {
+      changed: true,
+      count: sorted.length,
+      updated_at: now,
+      cache_version: nextVersion,
+    };
+  }
+
+  function markSuccessNoChange({ fullReconcile = false } = {}) {
+    const now = safeIsoNow();
+    if (fullReconcile) {
+      stmts.markSuccessNoChangeFull.run(now, null, now);
+    } else {
+      stmts.markSuccessNoChange.run(now, null);
+    }
+    return {
+      changed: false,
+      count: readCachedTasks().tasks.length,
+      updated_at: readCachedTasks().updated_at,
+      cache_version: getSyncState().cache_version || 0,
+    };
+  }
+
+  function markError(error) {
+    const message = String(error?.message || error || 'Unknown refresh error').slice(0, 1000);
+    stmts.markError.run(message);
+  }
+
+  async function buildTasksPageFromApi(top, skip = 0) {
+    const s = getSettings();
+    if (!s.api_key) throw new Error('API ключ не задан.');
+
+    const executorFilter = [
+      'executorgroup eq 243025',
+      'executor eq 2227907',
+      'executor eq 525111',
+      'executor eq 631481',
+      'executor eq 2222953',
+    ].join(' or ');
+
+    const topParam = Math.max(1, top);
+    const skipParam = Math.max(0, skip);
+    const fullFilter = `(${executorFilter})`;
+    const url = `https://apigw.intradesk.ru/tasklist/odata/v3/tasks?ApiKey=${s.api_key}`
+      + `&$filter=${encodeURIComponent(`(${fullFilter})`)}`
+      + `&$orderby=updatedat%20desc&$top=${topParam}&$skip=${skipParam}`;
+
+    logger.log(`[IntraDesk] GET top=${topParam} skip=${skipParam}`, url.replace(s.api_key, '***'));
+    const response = await fetch(url, { timeout: 30000 });
+    logger.log('[IntraDesk] Status:', response.status, response.statusText);
+    if (!response.ok) throw new Error(`IntraDesk вернул ${response.status}: ${response.statusText}`);
+
+    const json = await response.json();
+    const tasks = json.value || [];
+    const dictMap = {};
+    (json.dictionaries || []).forEach((d) => { dictMap[d.id] = d.name; });
+
+    tasks.forEach((task) => {
+      if (task.clientid && dictMap[task.clientid]) task._clientname = dictMap[task.clientid];
+      if (task.executor && dictMap[task.executor]) task._executorname = dictMap[task.executor];
+      if (task.executorgroup && dictMap[task.executorgroup]) task._executorgroupname = dictMap[task.executorgroup];
+      const statusKey = task.statusid || task.status;
+      if (statusKey && dictMap[statusKey]) task._statusname = dictMap[statusKey];
+    });
+
+    return tasks;
+  }
+
+  async function fetchAllTasksPaged(pageSize) {
+    const byId = new Map();
+    let skip = 0;
+    let pages = 0;
+    while (pages < MAX_SCAN_PAGES) {
+      const page = await buildTasksPageFromApi(pageSize, skip);
+      pages += 1;
+      if (!page.length) break;
+      for (const task of page) {
+        byId.set(extractTaskId(task), task);
+      }
+      if (page.length < pageSize) break;
+      skip += pageSize;
+    }
+    return sortByUpdatedDesc([...byId.values()]);
+  }
+
+  async function runIncrementalScan(reason = 'manual_incremental') {
+    const settings = normalizeSettings();
+    const pageSize = settings.scan_page_size;
+    const { tasks: existing } = readCachedTasks();
+    const byId = new Map(existing.map((t) => [extractTaskId(t), t]));
+
+    let changed = false;
+    let skip = 0;
+    let pages = 0;
+
+    while (pages < MAX_SCAN_PAGES) {
+      const page = await buildTasksPageFromApi(pageSize, skip);
+      pages += 1;
+
+      if (!page.length) break;
+
+      let pageChanged = false;
+      for (const task of page) {
+        const taskId = extractTaskId(task);
+        const previous = byId.get(taskId);
+        if (!previous || extractUpdatedAt(previous) !== extractUpdatedAt(task)) {
+          byId.set(taskId, task);
+          changed = true;
+          pageChanged = true;
+        }
+      }
+
+      if (!pageChanged) break;
+      if (page.length < pageSize) break;
+      skip += pageSize;
+    }
+
+    if (!changed) {
+      logger.log(`[Refresh] Incremental scan: no changes (${reason})`);
+      return markSuccessNoChange({ fullReconcile: false });
+    }
+
+    logger.log(`[Refresh] Incremental scan: changed (${reason})`);
+    return writeCacheAndIndexes([...byId.values()], { fullReconcile: false });
+  }
+
+  async function runFullReconcile(reason = 'manual_full_reconcile') {
+    const settings = normalizeSettings();
+    const pageSize = settings.scan_page_size;
+    const nextTasks = await fetchAllTasksPaged(pageSize);
+    const currentTasks = readCachedTasks().tasks;
+
+    if (!hasChangesByIdAndUpdated(currentTasks, nextTasks)) {
+      logger.log(`[Refresh] Full reconcile: no changes (${reason})`);
+      return markSuccessNoChange({ fullReconcile: true });
+    }
+
+    logger.log(`[Refresh] Full reconcile: changed (${reason})`);
+    return writeCacheAndIndexes(nextTasks, { fullReconcile: true });
+  }
+
+  function removeTaskFromCache(taskId, reason = 'webhook_remove') {
+    const id = String(taskId || '').trim();
+    if (!id) return markSuccessNoChange({ fullReconcile: false });
+
+    const { tasks: existing } = readCachedTasks();
+    const filtered = existing.filter((t) => extractTaskId(t) !== id);
+    if (filtered.length === existing.length) {
+      logger.log(`[Refresh] Remove task: not found ${id} (${reason})`);
+      return markSuccessNoChange({ fullReconcile: false });
+    }
+
+    logger.log(`[Refresh] Remove task: removed ${id} (${reason})`);
+    return writeCacheAndIndexes(filtered, { fullReconcile: false });
+  }
+
+  function shouldRunDailyFull(settings, syncState, now = new Date()) {
+    const hour = settings.daily_full_refresh_hour;
+    if (now.getHours() < hour) return false;
+
+    const last = syncState.last_full_reconcile_at ? new Date(syncState.last_full_reconcile_at) : null;
+    if (!last || Number.isNaN(last.getTime())) return true;
+
+    const dayKeyNow = `${now.getFullYear()}-${now.getMonth()}-${now.getDate()}`;
+    const dayKeyLast = `${last.getFullYear()}-${last.getMonth()}-${last.getDate()}`;
+    return dayKeyNow !== dayKeyLast;
+  }
+
+  let running = false;
+  let pending = null;
+  let schedulerTimer = null;
+
+  function jobPriority(type) {
+    if (type === 'full') return 3;
+    if (type === 'incremental') return 2;
+    return 1; // remove
+  }
+
+  function createPending(job) {
+    const deferred = {};
+    deferred.promise = new Promise((resolve, reject) => {
+      deferred.resolve = resolve;
+      deferred.reject = reject;
+    });
+    return {
+      type: job.type,
+      reason: job.reason,
+      removeIds: new Set(job.removeIds || []),
+      deferred,
+    };
+  }
+
+  function mergePending(existing, incoming) {
+    const merged = existing;
+    if (jobPriority(incoming.type) > jobPriority(merged.type)) {
+      merged.type = incoming.type;
+      if (merged.type === 'full') merged.removeIds = new Set();
+    }
+    if (incoming.reason) merged.reason = incoming.reason;
+    if (incoming.removeIds) {
+      for (const id of incoming.removeIds) merged.removeIds.add(id);
+    }
+    return merged;
+  }
+
+  async function executeJob(job) {
+    stmts.setInProgress.run(1);
+    try {
+      if (job.type === 'full') return await runFullReconcile(job.reason);
+      if (job.type === 'incremental') {
+        if (job.removeIds && job.removeIds.size) {
+          for (const id of job.removeIds) removeTaskFromCache(id, `${job.reason}:queued-remove`);
+        }
+        return await runIncrementalScan(job.reason);
+      }
+      if (job.type === 'remove') {
+        let lastResult = null;
+        for (const id of job.removeIds || []) {
+          lastResult = removeTaskFromCache(id, job.reason);
+        }
+        return lastResult || markSuccessNoChange({ fullReconcile: false });
+      }
+      return markSuccessNoChange({ fullReconcile: false });
+    } catch (error) {
+      markError(error);
+      throw error;
+    } finally {
+      stmts.setInProgress.run(0);
+    }
+  }
+
+  function runJobAndDrain(job) {
+    running = true;
+    return (async () => {
+      let result;
+      let error;
+      try {
+        result = await executeJob(job);
+      } catch (e) {
+        error = e;
+      }
+      running = false;
+
+      if (pending) {
+        const next = pending;
+        pending = null;
+        runJobAndDrain({
+          type: next.type,
+          reason: next.reason,
+          removeIds: next.removeIds,
+        }).then(next.deferred.resolve, next.deferred.reject);
+      }
+
+      if (error) throw error;
+      return result;
+    })();
+  }
+
+  function requestJob(job, { wait = true } = {}) {
+    const normalized = {
+      type: job.type,
+      reason: job.reason || job.type,
+      removeIds: new Set(job.removeIds || []),
+    };
+
+    if (!running) {
+      const p = runJobAndDrain(normalized);
+      if (!wait) p.catch((e) => logger.error('[Refresh] background error:', e.message));
+      return wait ? p : Promise.resolve({ queued: false });
+    }
+
+    if (!pending) {
+      pending = createPending(normalized);
+    } else {
+      pending = mergePending(pending, normalized);
+    }
+
+    return wait ? pending.deferred.promise : Promise.resolve({ queued: true });
+  }
+
+  function requestIncremental(reason, opts) {
+    return requestJob({ type: 'incremental', reason }, opts);
+  }
+
+  function requestFull(reason, opts) {
+    return requestJob({ type: 'full', reason }, opts);
+  }
+
+  function requestRemoveTask(taskId, reason, opts) {
+    return requestJob({ type: 'remove', reason, removeIds: [String(taskId)] }, opts);
+  }
+
+  function parseWebhookAction(eventTypeRaw) {
+    const eventType = String(eventTypeRaw || '').toLowerCase().trim();
+    if (!eventType) return { action: 'ignore', eventType };
+    if (CHANGE_EVENT_TYPES.has(eventType)) return { action: 'incremental', eventType };
+    if (REMOVE_EVENT_TYPES.has(eventType)) return { action: 'remove', eventType };
+    return { action: 'ignore', eventType };
+  }
+
+  function extractWebhookTaskId(body = {}) {
+    return (
+      body.task_id
+      || body.taskId
+      || body.task?.id
+      || body.data?.task_id
+      || body.data?.taskId
+      || body.data?.task?.id
+      || null
+    );
+  }
+
+  function getMode() {
+    return normalizeSettings().refresh_mode;
+  }
+
+  function getStatusSnapshot() {
+    const sync = getSyncState();
+    return {
+      cache_version: Number(sync.cache_version || 0),
+      refresh_in_progress: !!sync.refresh_in_progress,
+      last_success_at: sync.last_success_at || null,
+      last_error: sync.last_error || null,
+      last_full_reconcile_at: sync.last_full_reconcile_at || null,
+    };
+  }
+
+  async function schedulerTick() {
+    const settings = normalizeSettings();
+    const state = getSyncState();
+
+    if (shouldRunDailyFull(settings, state)) {
+      requestFull('daily_full_reconcile', { wait: false });
+    }
+
+    if (settings.refresh_mode === 'incremental_scan') {
+      requestIncremental('scheduler_incremental_scan', { wait: false });
+    }
+  }
+
+  function startScheduler() {
+    if (schedulerTimer) clearInterval(schedulerTimer);
+    schedulerTimer = setInterval(() => {
+      schedulerTick().catch((e) => logger.error('[Refresh] scheduler error:', e.message));
+    }, SCHEDULER_TICK_MS);
+    schedulerTick().catch((e) => logger.error('[Refresh] scheduler bootstrap error:', e.message));
+  }
+
+  function stopScheduler() {
+    if (schedulerTimer) {
+      clearInterval(schedulerTimer);
+      schedulerTimer = null;
+    }
+  }
+
+  return {
+    readCachedTasks,
+    getSyncState,
+    getStatusSnapshot,
+    getMode,
+    normalizeSettings,
+    startScheduler,
+    stopScheduler,
+    requestIncremental,
+    requestFull,
+    requestRemoveTask,
+    parseWebhookAction,
+    extractWebhookTaskId,
+  };
+}
+
+module.exports = {
+  createRefreshService,
+};

server.js

@@ -7,6 +7,8 @@ const bcrypt   = require('bcryptjs');
 const jwt      = require('jsonwebtoken');
 const fetch    = require('node-fetch');
 const path     = require('path');
+const crypto   = require('crypto');
+const { createRefreshService } = require('./refresh-service');
 
 const DEFAULT_JWT_SECRET = 'CHANGE_ME_IN_PRODUCTION_PLEASE';
 const DEFAULT_ADMIN_SECRET = 'admin_setup_secret';
@@ -99,7 +101,12 @@ db.exec(`
 `);
 
 // ── Middleware ────────────────────────────────────────────────────
-app.use(express.json({ limit: '10mb' }));
+app.use(express.json({
+  limit: '10mb',
+  verify: (req, _res, buf) => {
+    req.rawBody = buf;
+  },
+}));
 app.use(express.static(path.join(__dirname, 'public')));
 
 function auth(req, res, next) {
@@ -127,19 +134,27 @@ function parseMentions(text) {
 }
 // Ищем task_id по tasknumber из кэша
 function taskIdByNumber(taskNumber) {
+  const row = db.prepare('SELECT task_id FROM tasks_index WHERE task_number=?').get(String(taskNumber));
+  if (row?.task_id) return String(row.task_id);
   const cache = db.prepare('SELECT data FROM tasks_cache WHERE id=1').get();
   if (!cache) return null;
-  const tasks = JSON.parse(cache.data);
-  const t = tasks.find(t => String(t.tasknumber) === String(taskNumber));
-  return t ? String(t.id) : null;
+  try {
+    const tasks = JSON.parse(cache.data);
+    const t = tasks.find((x) => String(x.tasknumber) === String(taskNumber));
+    return t ? String(t.id) : null;
+  } catch {
+    return null;
+  }
 }
 // Возвращает tasknumber по task_id из кэша
 function taskNumberById(taskId) {
+  const row = db.prepare('SELECT task_number FROM tasks_index WHERE task_id=?').get(String(taskId));
+  if (row?.task_number) return String(row.task_number);
   try {
     const cache = db.prepare('SELECT data FROM tasks_cache WHERE id=1').get();
     if (!cache) return taskId;
     const tasks = JSON.parse(cache.data);
-    const t = tasks.find(t => String(t.id) === String(taskId));
+    const t = tasks.find((x) => String(x.id) === String(taskId));
     return t ? String(t.tasknumber || t.id) : taskId;
   } catch { return taskId; }
 }
@@ -169,6 +184,32 @@ function sseSendTo(u, event, data) {
   sseClients.get(key)?.forEach(r => { try { r.write(msg); } catch {} });
 }
 
+const refreshService = createRefreshService({
+  db,
+  fetch,
+  getSettings,
+  sseBroadcast,
+  logger: console,
+});
+
+function normalizeSignature(sigValue) {
+  const raw = String(sigValue || '').trim();
+  if (!raw) return '';
+  if (raw.startsWith('sha256=')) return raw.slice('sha256='.length).toLowerCase();
+  return raw.toLowerCase();
+}
+
+function verifyWebhookSignature(rawBody, secret, signatureHeader) {
+  const signature = normalizeSignature(signatureHeader);
+  if (!signature) return false;
+  const body = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(rawBody || '');
+  const expected = crypto.createHmac('sha256', secret).update(body).digest('hex');
+  const expectedBuf = Buffer.from(expected, 'hex');
+  const actualBuf = Buffer.from(signature, 'hex');
+  if (!expectedBuf.length || expectedBuf.length !== actualBuf.length) return false;
+  return crypto.timingSafeEqual(expectedBuf, actualBuf);
+}
+
 app.get('/healthz', (req, res) => {
   try {
     db.prepare('SELECT 1').get();
@@ -247,34 +288,87 @@ app.put('/api/users/:id/password', auth, adminOnly, (req, res) => {
 app.get('/api/status', auth, (req, res) => {
   const cache = db.prepare('SELECT updated_at FROM tasks_cache WHERE id=1').get();
   const s = getSettings();
+  const sync = refreshService.getStatusSnapshot();
   res.json({
     last_updated:          cache?.updated_at || null,
     auto_refresh_interval: parseInt(s.auto_refresh_interval) || 300,
     unread_mode:           s.unread_mode || 'highlight',
     export_fields:         s.export_fields || '',
     export_status_mode:    s.export_status_mode || 'A',
+    cache_version:         sync.cache_version,
+    refresh_in_progress:   sync.refresh_in_progress,
+    last_success_at:       sync.last_success_at,
+    last_error:            sync.last_error,
+    last_full_reconcile_at: sync.last_full_reconcile_at,
+    refresh_mode:          refreshService.getMode(),
   });
 });
 
 // ── Tasks ─────────────────────────────────────────────────────────
 app.get('/api/tasks', auth, (req, res) => {
   const cache = db.prepare('SELECT * FROM tasks_cache WHERE id=1').get();
-  if (!cache) return res.json({ tasks: [], updated_at: null });
-  res.json({ tasks: JSON.parse(cache.data), updated_at: cache.updated_at });
+  const sync = refreshService.getStatusSnapshot();
+  if (!cache) return res.json({ tasks: [], updated_at: null, cache_version: sync.cache_version });
+  res.json({ tasks: JSON.parse(cache.data), updated_at: cache.updated_at, cache_version: sync.cache_version });
 });
 
 app.post('/api/tasks/refresh', auth, async (req, res) => {
-  try { res.json({ ok: true, count: await fetchAndCacheTasks(), updated_at: new Date().toISOString() }); }
+  try {
+    const result = await refreshService.requestIncremental('manual_api_refresh', { wait: true });
+    res.json({ ok: true, ...result });
+  }
   catch (e) { console.error('Refresh error:', e.message); res.status(500).json({ error: e.message }); }
 });
 
 app.post('/api/tasks/refresh/full', auth, adminOnly, async (req, res) => {
   try {
-    const count = await fetchAndCacheAllTasks();
-    res.json({ ok: true, count, updated_at: new Date().toISOString() });
+    const result = await refreshService.requestFull('manual_api_full_refresh', { wait: true });
+    res.json({ ok: true, ...result });
   } catch (e) { console.error('Full refresh error:', e.message); res.status(500).json({ error: e.message }); }
 });
 
+app.post('/api/webhooks/intradesk', async (req, res) => {
+  try {
+    if (refreshService.getMode() !== 'webhook_daily') {
+      return res.status(202).json({ ok: true, ignored: true, reason: 'refresh_mode_not_webhook_daily' });
+    }
+
+    const s = getSettings();
+    const webhookSecret = s.webhook_secret || process.env.WEBHOOK_SECRET || '';
+    if (!webhookSecret) return res.status(503).json({ error: 'Webhook secret is not configured' });
+
+    const signature = req.headers['x-webhook-signature'] || req.headers['x-signature'] || '';
+    if (!verifyWebhookSignature(req.rawBody, webhookSecret, signature)) {
+      return res.status(401).json({ error: 'Invalid webhook signature' });
+    }
+
+    const body = req.body || {};
+    const eventTypeRaw = body.event_type || body.eventType || body.type || body.event || '';
+    const { action, eventType } = refreshService.parseWebhookAction(eventTypeRaw);
+    const taskId = refreshService.extractWebhookTaskId(body);
+
+    if (action === 'incremental') {
+      await refreshService.requestIncremental(`webhook:${eventType}`, { wait: false });
+      return res.json({ ok: true, action: 'incremental_queued' });
+    }
+
+    if (action === 'remove') {
+      if (taskId) {
+        await refreshService.requestRemoveTask(taskId, `webhook:${eventType}`, { wait: false });
+        return res.json({ ok: true, action: 'task_removed_or_queued', task_id: String(taskId) });
+      }
+      await refreshService.requestFull(`webhook:${eventType}:no_task_id`, { wait: false });
+      return res.json({ ok: true, action: 'full_reconcile_queued' });
+    }
+
+    console.log(`[Webhook] Unknown event type: ${eventTypeRaw}`);
+    return res.json({ ok: true, ignored: true, reason: 'unknown_event_type', event_type: eventTypeRaw });
+  } catch (e) {
+    console.error('[Webhook] Error:', e.message);
+    return res.status(500).json({ error: e.message });
+  }
+});
+
 // ── Marks ─────────────────────────────────────────────────────────
 const _marksConfig = require('./public/js/statuses.js');
 const ALLOWED_MARKS = new Set(_marksConfig.map(m => m.key));
@@ -443,6 +537,10 @@ const ALLOWED_SETTINGS = new Set([
   'status_ids',
   'event_types',
   'auto_refresh_interval',
+  'refresh_mode',
+  'scan_page_size',
+  'daily_full_refresh_hour',
+  'webhook_secret',
   'unread_mode',
   'export_fields',
   'export_status_mode',
@@ -454,17 +552,20 @@ app.get('/api/settings', auth, adminOnly, (req, res) => {
     status_ids:            s.status_ids || '68051,68046',
     event_types:           s.event_types || '',
     auto_refresh_interval: s.auto_refresh_interval || '300',
+    refresh_mode:          s.refresh_mode || 'incremental_scan',
+    scan_page_size:        s.scan_page_size || '50',
+    daily_full_refresh_hour: s.daily_full_refresh_hour || '4',
     unread_mode:           s.unread_mode || 'highlight',
     export_fields:         s.export_fields || '',
     export_status_mode:    s.export_status_mode || 'B',
     api_key_set:           !!s.api_key,
+    webhook_secret_set:    !!s.webhook_secret,
   });
 });
 
 app.post('/api/settings', auth, adminOnly, (req, res) => {
   const body = req.body || {};
   Object.entries(body).forEach(([k, v]) => { if (ALLOWED_SETTINGS.has(k) && v !== undefined) setSetting(k, String(v)); });
-  if (body.auto_refresh_interval !== undefined) setupAutoRefresh();
   res.json({ ok: true });
 });
 
@@ -521,77 +622,8 @@ app.get('/api/tasks/lastcomment/:taskId', auth, async (req, res) => {
 });
 
 // ── IntraDesk fetch ───────────────────────────────────────────────
-async function buildTasksFromApi(top) {
-  const s = getSettings();
-  if (!s.api_key) throw new Error('API ключ не задан.');
-  const executorFilter = [
-    'executorgroup eq 243025',  // Поддержка, 1-ая линия
-    'executor eq 2227907',      // Высоких Дмитрий
-    'executor eq 525111',       // Абрамов Сергей
-    'executor eq 631481',       // Железняков Антон
-    'executor eq 2222953',      // Коваленко Артем
-  ].join(' or ');
-  const fullFilter = `(${executorFilter})`;
-  const topParam = top > 0 ? top : 2000;
-  const url = `https://apigw.intradesk.ru/tasklist/odata/v3/tasks?ApiKey=${s.api_key}&$filter=${encodeURIComponent(`(${fullFilter})`)}&$orderby=updatedat%20desc&$top=${topParam}`;
-  console.log(`[IntraDesk] GET top=${topParam}`, url.replace(s.api_key, '***'));
-  const r = await fetch(url, { timeout: 30000 });
-  console.log('[IntraDesk] Status:', r.status, r.statusText);
-  if (!r.ok) throw new Error(`IntraDesk вернул ${r.status}: ${r.statusText}`);
-  const json = await r.json();
-  const tasks = json.value || [];
-  const dictMap = {};
-  (json.dictionaries || []).forEach(d => { dictMap[d.id] = d.name; });
-  tasks.forEach(task => {
-    if (task.clientid      && dictMap[task.clientid])      task._clientname        = dictMap[task.clientid];
-    if (task.executor      && dictMap[task.executor])      task._executorname      = dictMap[task.executor];
-    if (task.executorgroup && dictMap[task.executorgroup]) task._executorgroupname = dictMap[task.executorgroup];
-    // API возвращает статус в поле 'status' (числовой id), иногда 'statusid' — проверяем оба
-    const statusKey = task.statusid || task.status;
-    if (statusKey && dictMap[statusKey]) task._statusname = dictMap[statusKey];
-  });
-  return tasks;
-}
+refreshService.startScheduler();
 
-// Quick refresh: fetch top-50 recent tasks and merge into cache
-async function fetchAndCacheTasks() {
-  const fresh = await buildTasksFromApi(50);
-  const existing = (() => {
-    try { const c = db.prepare('SELECT data FROM tasks_cache WHERE id=1').get(); return c ? JSON.parse(c.data) : []; }
-    catch { return []; }
-  })();
-  const byId = new Map(existing.map(t => [t.id, t]));
-  fresh.forEach(t => byId.set(t.id, t));
-  const merged = [...byId.values()].sort((a, b) => (b.updatedat||'') > (a.updatedat||'') ? 1 : -1);
-  const now = new Date().toISOString();
-  db.prepare(`INSERT INTO tasks_cache(id,data,updated_at) VALUES(1,?,?)
-    ON CONFLICT(id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at`).run(JSON.stringify(merged), now);
-  sseBroadcast('tasks_updated', { updated_at: now, count: merged.length });
-  return merged.length;
-}
-
-// Full refresh: fetch all tasks and replace cache
-async function fetchAndCacheAllTasks() {
-  const tasks = await buildTasksFromApi(0);
-  const now = new Date().toISOString();
-  db.prepare(`INSERT INTO tasks_cache(id,data,updated_at) VALUES(1,?,?)
-    ON CONFLICT(id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at`).run(JSON.stringify(tasks), now);
-  sseBroadcast('tasks_updated', { updated_at: now, count: tasks.length });
-  return tasks.length;
-}
-
-// ── Auto-refresh ──────────────────────────────────────────────────
-let autoRefreshTimer = null;
-function setupAutoRefresh() {
-  if (autoRefreshTimer) { clearInterval(autoRefreshTimer); autoRefreshTimer = null; }
-  const interval = parseInt(getSettings().auto_refresh_interval) || 0;
-  if (interval <= 0) { console.log('[AutoRefresh] Выключен'); return; }
-  autoRefreshTimer = setInterval(async () => {
-    try { console.log(`[${new Date().toLocaleTimeString('ru-RU')}] AutoRefresh: ${await fetchAndCacheTasks()} заявок`); }
-    catch (e) { console.error('[AutoRefresh] Ошибка:', e.message); }
-  }, interval * 1000);
-  console.log(`[AutoRefresh] Каждые ${interval}с`);
-}
-setupAutoRefresh();
 
 app.listen(PORT, () => { console.log(`\nServer listening on http://localhost:${PORT}\n`); });
+