diff --git a/Dockerfile b/Dockerfile index aa6fa97..feb2b51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ RUN npm install --omit=dev --no-audit --no-fund \ && npm cache clean --force COPY server.js ./ +COPY refresh-service.js ./ COPY public/ ./public/ # Папка для базы данных @@ -31,4 +32,4 @@ ENV DB_PATH=/data/intradesk.db HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ CMD node -e "const http=require('http');const req=http.get('http://127.0.0.1:3000/healthz',res=>process.exit(res.statusCode===200?0:1));req.on('error',()=>process.exit(1));" -CMD ["node", "server.js"] \ No newline at end of file +CMD ["node", "server.js"] diff --git a/public/js/auth.js b/public/js/auth.js index c784aa7..2510b88 100644 --- a/public/js/auth.js +++ b/public/js/auth.js @@ -19,6 +19,7 @@ document.addEventListener('keydown',e=>{ function doLogout() { localStorage.removeItem('intradesk_token'); authToken=null; currentUser=null; + reloadInFlight=false; if(statusPollTimer) clearInterval(statusPollTimer); if(clientCountdownTimer) clearInterval(clientCountdownTimer); stopSSE(); @@ -51,7 +52,13 @@ async function init() { loadUsersTable(); } const status=await apiGet('/api/status'); - if(status){serverRefreshInterval=status.auto_refresh_interval||300;lastKnownUpdatedAt=status.last_updated;} + if(status){ + serverRefreshInterval=status.auto_refresh_interval||300; + lastKnownUpdatedAt=status.last_updated; + if(status.cache_version!==undefined&&status.cache_version!==null){ + lastKnownCacheVersion=Number(status.cache_version)||0; + } + } startSSE(); startStatusPolling(); } @@ -76,6 +83,9 @@ async function loadTasksFromServer(silent=false) { return {task,lastComment:lc}; }); lastKnownUpdatedAt=data.updated_at; + if(data.cache_version!==undefined&&data.cache_version!==null){ + lastKnownCacheVersion=Number(data.cache_version)||lastKnownCacheVersion; + } const el=id=>document.getElementById(id); if(el('total-loaded')) el('total-loaded').textContent=allData.length; if(el('total-count')) el('total-count').textContent=allData.length; @@ -146,18 +156,31 @@ async function loadAllTasks() { } async function reloadData() { - await Promise.all([loadMarks(),loadNotes()]); - await loadTasksFromServer(true); + if(reloadInFlight) return; + reloadInFlight=true; + try { + await Promise.all([loadMarks(),loadNotes()]); + await loadTasksFromServer(true); + } finally { + reloadInFlight=false; + } } function startStatusPolling() { if(statusPollTimer) clearInterval(statusPollTimer); statusPollTimer=setInterval(async()=>{ const s=await apiGet('/api/status'); + if(!s) return; + const serverVersion=Number(s.cache_version||0); + if(serverVersion&&serverVersion!==lastKnownCacheVersion){ + lastKnownCacheVersion=serverVersion; + if(s.last_updated) lastKnownUpdatedAt=s.last_updated; + await reloadData(); + return; + } if(s?.last_updated && s.last_updated!==lastKnownUpdatedAt) { lastKnownUpdatedAt=s.last_updated; - await Promise.all([loadMarks(),loadNotes()]); - await loadTasksFromServer(true); + await reloadData(); } },30000); } diff --git a/public/js/diff.js b/public/js/diff.js index 5ae96b1..03cbd34 100644 --- a/public/js/diff.js +++ b/public/js/diff.js @@ -136,6 +136,9 @@ function applyTasksDiff(newRawTasks, silent = false) { // Обновляем мета-данные (те же строки, что были в оригинале) lastKnownUpdatedAt = data.updated_at; + if (data.cache_version !== undefined && data.cache_version !== null) { + lastKnownCacheVersion = Number(data.cache_version) || lastKnownCacheVersion; + } const el = id => document.getElementById(id); if (el('total-loaded')) el('total-loaded').textContent = allData.length; if (el('total-count')) el('total-count').textContent = allData.length; diff --git a/public/js/sse.js b/public/js/sse.js index e14af21..e6416cb 100644 --- a/public/js/sse.js +++ b/public/js/sse.js @@ -86,6 +86,9 @@ function startSSE() { sseSource.addEventListener('tasks_updated',e=>{ const d=JSON.parse(e.data); + const incomingVersion=Number(d.cache_version||0); + if(incomingVersion&&incomingVersion===lastKnownCacheVersion) return; + if(incomingVersion) lastKnownCacheVersion=incomingVersion; showToast(`🔄 Обновление: ${d.count} заявок`,'info',4000); reloadData(); }); diff --git a/public/js/state.js b/public/js/state.js index 7e358c3..554c3bb 100644 --- a/public/js/state.js +++ b/public/js/state.js @@ -25,6 +25,8 @@ let mentionsData = []; let mentionUnread = 0; let knownUsers = []; let lastKnownUpdatedAt = null; +let lastKnownCacheVersion = 0; +let reloadInFlight = false; let statusPollTimer = null, clientCountdownTimer = null, clientCountdownSec = 0; let serverRefreshInterval = 300; let commentCounts = {}; diff --git a/refresh-service.js b/refresh-service.js new file mode 100644 index 0000000..dece87a --- /dev/null +++ b/refresh-service.js @@ -0,0 +1,576 @@ +'use strict'; + +const DEFAULT_REFRESH_MODE = 'incremental_scan'; +const DEFAULT_SCAN_PAGE_SIZE = 50; +const DEFAULT_DAILY_FULL_REFRESH_HOUR = 4; +const SCHEDULER_TICK_MS = 60 * 1000; +const MAX_SCAN_PAGES = 500; + +const CHANGE_EVENT_TYPES = new Set([ + 'created', + 'updated', + 'comment', + 'status', + 'executor_changed', +]); + +const REMOVE_EVENT_TYPES = new Set([ + 'archived', + 'out_of_scope', + 'deleted', +]); + +function safeIsoNow() { + return new Date().toISOString(); +} + +function toInt(value, fallback) { + const n = parseInt(value, 10); + return Number.isFinite(n) ? n : fallback; +} + +function normalizeRefreshMode(mode) { + return mode === 'webhook_daily' ? 'webhook_daily' : DEFAULT_REFRESH_MODE; +} + +function normalizePageSize(value) { + const n = toInt(value, DEFAULT_SCAN_PAGE_SIZE); + return Math.max(10, Math.min(500, n)); +} + +function normalizeDailyHour(value) { + const n = toInt(value, DEFAULT_DAILY_FULL_REFRESH_HOUR); + return Math.max(0, Math.min(23, n)); +} + +function sortByUpdatedDesc(tasks) { + return [...tasks].sort((a, b) => { + const au = a?.updatedat || ''; + const bu = b?.updatedat || ''; + if (bu === au) return 0; + return bu > au ? 1 : -1; + }); +} + +function extractTaskId(task) { + return String(task?.id ?? ''); +} + +function extractTaskNumber(task) { + return String(task?.tasknumber ?? task?.id ?? ''); +} + +function extractUpdatedAt(task) { + return String(task?.updatedat ?? ''); +} + +function hasChangesByIdAndUpdated(oldTasks, newTasks) { + if (oldTasks.length !== newTasks.length) return true; + const oldMap = new Map(oldTasks.map((t) => [extractTaskId(t), extractUpdatedAt(t)])); + for (const task of newTasks) { + const id = extractTaskId(task); + if (!oldMap.has(id)) return true; + if (oldMap.get(id) !== extractUpdatedAt(task)) return true; + } + return false; +} + +function createRefreshService({ db, fetch, getSettings, sseBroadcast, logger = console }) { + db.exec(` + CREATE TABLE IF NOT EXISTS sync_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + cache_version INTEGER NOT NULL DEFAULT 0, + refresh_in_progress INTEGER NOT NULL DEFAULT 0, + last_success_at TEXT, + last_error TEXT, + last_full_reconcile_at TEXT + ); + + INSERT OR IGNORE INTO sync_state(id, cache_version, refresh_in_progress) + VALUES(1, 0, 0); + + CREATE TABLE IF NOT EXISTS tasks_index ( + task_id TEXT PRIMARY KEY, + task_number TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_tasks_index_number ON tasks_index(task_number); + `); + + const stmts = { + getCacheRow: db.prepare('SELECT data, updated_at FROM tasks_cache WHERE id=1'), + upsertCache: db.prepare(` + INSERT INTO tasks_cache(id,data,updated_at) VALUES(1,?,?) + ON CONFLICT(id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at + `), + getSyncState: db.prepare(` + SELECT cache_version, refresh_in_progress, last_success_at, last_error, last_full_reconcile_at + FROM sync_state WHERE id=1 + `), + setInProgress: db.prepare('UPDATE sync_state SET refresh_in_progress=? WHERE id=1'), + markError: db.prepare(` + UPDATE sync_state + SET refresh_in_progress=0, last_error=? + WHERE id=1 + `), + markSuccessNoChange: db.prepare(` + UPDATE sync_state + SET refresh_in_progress=0, last_success_at=?, last_error=? + WHERE id=1 + `), + markSuccessNoChangeFull: db.prepare(` + UPDATE sync_state + SET refresh_in_progress=0, last_success_at=?, last_error=?, last_full_reconcile_at=? + WHERE id=1 + `), + writeStateAfterCache: db.prepare(` + UPDATE sync_state + SET cache_version=?, refresh_in_progress=0, last_success_at=?, last_error=? + WHERE id=1 + `), + writeStateAfterFull: db.prepare(` + UPDATE sync_state + SET cache_version=?, refresh_in_progress=0, last_success_at=?, last_error=?, last_full_reconcile_at=? + WHERE id=1 + `), + clearIndex: db.prepare('DELETE FROM tasks_index'), + insertIndex: db.prepare('INSERT INTO tasks_index(task_id, task_number, updated_at) VALUES(?,?,?)'), + }; + + function getSyncState() { + const row = stmts.getSyncState.get(); + if (row) return row; + return { + cache_version: 0, + refresh_in_progress: 0, + last_success_at: null, + last_error: null, + last_full_reconcile_at: null, + }; + } + + function readCachedTasks() { + try { + const row = stmts.getCacheRow.get(); + if (!row) return { tasks: [], updated_at: null }; + return { + tasks: JSON.parse(row.data || '[]'), + updated_at: row.updated_at || null, + }; + } catch { + return { tasks: [], updated_at: null }; + } + } + + function normalizeSettings() { + const s = getSettings(); + return { + ...s, + refresh_mode: normalizeRefreshMode(s.refresh_mode), + scan_page_size: normalizePageSize(s.scan_page_size), + daily_full_refresh_hour: normalizeDailyHour(s.daily_full_refresh_hour), + }; + } + + function writeCacheAndIndexes(tasks, { fullReconcile = false } = {}) { + const now = safeIsoNow(); + const state = getSyncState(); + const nextVersion = Number(state.cache_version || 0) + 1; + const sorted = sortByUpdatedDesc(tasks); + + const tx = db.transaction(() => { + stmts.upsertCache.run(JSON.stringify(sorted), now); + stmts.clearIndex.run(); + for (const task of sorted) { + const taskId = extractTaskId(task); + if (!taskId) continue; + stmts.insertIndex.run(taskId, extractTaskNumber(task), extractUpdatedAt(task) || now); + } + if (fullReconcile) { + stmts.writeStateAfterFull.run(nextVersion, now, null, now); + } else { + stmts.writeStateAfterCache.run(nextVersion, now, null); + } + }); + tx(); + + sseBroadcast('tasks_updated', { + updated_at: now, + count: sorted.length, + cache_version: nextVersion, + }); + + return { + changed: true, + count: sorted.length, + updated_at: now, + cache_version: nextVersion, + }; + } + + function markSuccessNoChange({ fullReconcile = false } = {}) { + const now = safeIsoNow(); + if (fullReconcile) { + stmts.markSuccessNoChangeFull.run(now, null, now); + } else { + stmts.markSuccessNoChange.run(now, null); + } + return { + changed: false, + count: readCachedTasks().tasks.length, + updated_at: readCachedTasks().updated_at, + cache_version: getSyncState().cache_version || 0, + }; + } + + function markError(error) { + const message = String(error?.message || error || 'Unknown refresh error').slice(0, 1000); + stmts.markError.run(message); + } + + async function buildTasksPageFromApi(top, skip = 0) { + const s = getSettings(); + if (!s.api_key) throw new Error('API ключ не задан.'); + + const executorFilter = [ + 'executorgroup eq 243025', + 'executor eq 2227907', + 'executor eq 525111', + 'executor eq 631481', + 'executor eq 2222953', + ].join(' or '); + + const topParam = Math.max(1, top); + const skipParam = Math.max(0, skip); + const fullFilter = `(${executorFilter})`; + const url = `https://apigw.intradesk.ru/tasklist/odata/v3/tasks?ApiKey=${s.api_key}` + + `&$filter=${encodeURIComponent(`(${fullFilter})`)}` + + `&$orderby=updatedat%20desc&$top=${topParam}&$skip=${skipParam}`; + + logger.log(`[IntraDesk] GET top=${topParam} skip=${skipParam}`, url.replace(s.api_key, '***')); + const response = await fetch(url, { timeout: 30000 }); + logger.log('[IntraDesk] Status:', response.status, response.statusText); + if (!response.ok) throw new Error(`IntraDesk вернул ${response.status}: ${response.statusText}`); + + const json = await response.json(); + const tasks = json.value || []; + const dictMap = {}; + (json.dictionaries || []).forEach((d) => { dictMap[d.id] = d.name; }); + + tasks.forEach((task) => { + if (task.clientid && dictMap[task.clientid]) task._clientname = dictMap[task.clientid]; + if (task.executor && dictMap[task.executor]) task._executorname = dictMap[task.executor]; + if (task.executorgroup && dictMap[task.executorgroup]) task._executorgroupname = dictMap[task.executorgroup]; + const statusKey = task.statusid || task.status; + if (statusKey && dictMap[statusKey]) task._statusname = dictMap[statusKey]; + }); + + return tasks; + } + + async function fetchAllTasksPaged(pageSize) { + const byId = new Map(); + let skip = 0; + let pages = 0; + while (pages < MAX_SCAN_PAGES) { + const page = await buildTasksPageFromApi(pageSize, skip); + pages += 1; + if (!page.length) break; + for (const task of page) { + byId.set(extractTaskId(task), task); + } + if (page.length < pageSize) break; + skip += pageSize; + } + return sortByUpdatedDesc([...byId.values()]); + } + + async function runIncrementalScan(reason = 'manual_incremental') { + const settings = normalizeSettings(); + const pageSize = settings.scan_page_size; + const { tasks: existing } = readCachedTasks(); + const byId = new Map(existing.map((t) => [extractTaskId(t), t])); + + let changed = false; + let skip = 0; + let pages = 0; + + while (pages < MAX_SCAN_PAGES) { + const page = await buildTasksPageFromApi(pageSize, skip); + pages += 1; + + if (!page.length) break; + + let pageChanged = false; + for (const task of page) { + const taskId = extractTaskId(task); + const previous = byId.get(taskId); + if (!previous || extractUpdatedAt(previous) !== extractUpdatedAt(task)) { + byId.set(taskId, task); + changed = true; + pageChanged = true; + } + } + + if (!pageChanged) break; + if (page.length < pageSize) break; + skip += pageSize; + } + + if (!changed) { + logger.log(`[Refresh] Incremental scan: no changes (${reason})`); + return markSuccessNoChange({ fullReconcile: false }); + } + + logger.log(`[Refresh] Incremental scan: changed (${reason})`); + return writeCacheAndIndexes([...byId.values()], { fullReconcile: false }); + } + + async function runFullReconcile(reason = 'manual_full_reconcile') { + const settings = normalizeSettings(); + const pageSize = settings.scan_page_size; + const nextTasks = await fetchAllTasksPaged(pageSize); + const currentTasks = readCachedTasks().tasks; + + if (!hasChangesByIdAndUpdated(currentTasks, nextTasks)) { + logger.log(`[Refresh] Full reconcile: no changes (${reason})`); + return markSuccessNoChange({ fullReconcile: true }); + } + + logger.log(`[Refresh] Full reconcile: changed (${reason})`); + return writeCacheAndIndexes(nextTasks, { fullReconcile: true }); + } + + function removeTaskFromCache(taskId, reason = 'webhook_remove') { + const id = String(taskId || '').trim(); + if (!id) return markSuccessNoChange({ fullReconcile: false }); + + const { tasks: existing } = readCachedTasks(); + const filtered = existing.filter((t) => extractTaskId(t) !== id); + if (filtered.length === existing.length) { + logger.log(`[Refresh] Remove task: not found ${id} (${reason})`); + return markSuccessNoChange({ fullReconcile: false }); + } + + logger.log(`[Refresh] Remove task: removed ${id} (${reason})`); + return writeCacheAndIndexes(filtered, { fullReconcile: false }); + } + + function shouldRunDailyFull(settings, syncState, now = new Date()) { + const hour = settings.daily_full_refresh_hour; + if (now.getHours() < hour) return false; + + const last = syncState.last_full_reconcile_at ? new Date(syncState.last_full_reconcile_at) : null; + if (!last || Number.isNaN(last.getTime())) return true; + + const dayKeyNow = `${now.getFullYear()}-${now.getMonth()}-${now.getDate()}`; + const dayKeyLast = `${last.getFullYear()}-${last.getMonth()}-${last.getDate()}`; + return dayKeyNow !== dayKeyLast; + } + + let running = false; + let pending = null; + let schedulerTimer = null; + + function jobPriority(type) { + if (type === 'full') return 3; + if (type === 'incremental') return 2; + return 1; // remove + } + + function createPending(job) { + const deferred = {}; + deferred.promise = new Promise((resolve, reject) => { + deferred.resolve = resolve; + deferred.reject = reject; + }); + return { + type: job.type, + reason: job.reason, + removeIds: new Set(job.removeIds || []), + deferred, + }; + } + + function mergePending(existing, incoming) { + const merged = existing; + if (jobPriority(incoming.type) > jobPriority(merged.type)) { + merged.type = incoming.type; + if (merged.type === 'full') merged.removeIds = new Set(); + } + if (incoming.reason) merged.reason = incoming.reason; + if (incoming.removeIds) { + for (const id of incoming.removeIds) merged.removeIds.add(id); + } + return merged; + } + + async function executeJob(job) { + stmts.setInProgress.run(1); + try { + if (job.type === 'full') return await runFullReconcile(job.reason); + if (job.type === 'incremental') { + if (job.removeIds && job.removeIds.size) { + for (const id of job.removeIds) removeTaskFromCache(id, `${job.reason}:queued-remove`); + } + return await runIncrementalScan(job.reason); + } + if (job.type === 'remove') { + let lastResult = null; + for (const id of job.removeIds || []) { + lastResult = removeTaskFromCache(id, job.reason); + } + return lastResult || markSuccessNoChange({ fullReconcile: false }); + } + return markSuccessNoChange({ fullReconcile: false }); + } catch (error) { + markError(error); + throw error; + } finally { + stmts.setInProgress.run(0); + } + } + + function runJobAndDrain(job) { + running = true; + return (async () => { + let result; + let error; + try { + result = await executeJob(job); + } catch (e) { + error = e; + } + running = false; + + if (pending) { + const next = pending; + pending = null; + runJobAndDrain({ + type: next.type, + reason: next.reason, + removeIds: next.removeIds, + }).then(next.deferred.resolve, next.deferred.reject); + } + + if (error) throw error; + return result; + })(); + } + + function requestJob(job, { wait = true } = {}) { + const normalized = { + type: job.type, + reason: job.reason || job.type, + removeIds: new Set(job.removeIds || []), + }; + + if (!running) { + const p = runJobAndDrain(normalized); + if (!wait) p.catch((e) => logger.error('[Refresh] background error:', e.message)); + return wait ? p : Promise.resolve({ queued: false }); + } + + if (!pending) { + pending = createPending(normalized); + } else { + pending = mergePending(pending, normalized); + } + + return wait ? pending.deferred.promise : Promise.resolve({ queued: true }); + } + + function requestIncremental(reason, opts) { + return requestJob({ type: 'incremental', reason }, opts); + } + + function requestFull(reason, opts) { + return requestJob({ type: 'full', reason }, opts); + } + + function requestRemoveTask(taskId, reason, opts) { + return requestJob({ type: 'remove', reason, removeIds: [String(taskId)] }, opts); + } + + function parseWebhookAction(eventTypeRaw) { + const eventType = String(eventTypeRaw || '').toLowerCase().trim(); + if (!eventType) return { action: 'ignore', eventType }; + if (CHANGE_EVENT_TYPES.has(eventType)) return { action: 'incremental', eventType }; + if (REMOVE_EVENT_TYPES.has(eventType)) return { action: 'remove', eventType }; + return { action: 'ignore', eventType }; + } + + function extractWebhookTaskId(body = {}) { + return ( + body.task_id + || body.taskId + || body.task?.id + || body.data?.task_id + || body.data?.taskId + || body.data?.task?.id + || null + ); + } + + function getMode() { + return normalizeSettings().refresh_mode; + } + + function getStatusSnapshot() { + const sync = getSyncState(); + return { + cache_version: Number(sync.cache_version || 0), + refresh_in_progress: !!sync.refresh_in_progress, + last_success_at: sync.last_success_at || null, + last_error: sync.last_error || null, + last_full_reconcile_at: sync.last_full_reconcile_at || null, + }; + } + + async function schedulerTick() { + const settings = normalizeSettings(); + const state = getSyncState(); + + if (shouldRunDailyFull(settings, state)) { + requestFull('daily_full_reconcile', { wait: false }); + } + + if (settings.refresh_mode === 'incremental_scan') { + requestIncremental('scheduler_incremental_scan', { wait: false }); + } + } + + function startScheduler() { + if (schedulerTimer) clearInterval(schedulerTimer); + schedulerTimer = setInterval(() => { + schedulerTick().catch((e) => logger.error('[Refresh] scheduler error:', e.message)); + }, SCHEDULER_TICK_MS); + schedulerTick().catch((e) => logger.error('[Refresh] scheduler bootstrap error:', e.message)); + } + + function stopScheduler() { + if (schedulerTimer) { + clearInterval(schedulerTimer); + schedulerTimer = null; + } + } + + return { + readCachedTasks, + getSyncState, + getStatusSnapshot, + getMode, + normalizeSettings, + startScheduler, + stopScheduler, + requestIncremental, + requestFull, + requestRemoveTask, + parseWebhookAction, + extractWebhookTaskId, + }; +} + +module.exports = { + createRefreshService, +}; diff --git a/server.js b/server.js index 35116d6..cce5e48 100644 --- a/server.js +++ b/server.js @@ -7,6 +7,8 @@ const bcrypt = require('bcryptjs'); const jwt = require('jsonwebtoken'); const fetch = require('node-fetch'); const path = require('path'); +const crypto = require('crypto'); +const { createRefreshService } = require('./refresh-service'); const DEFAULT_JWT_SECRET = 'CHANGE_ME_IN_PRODUCTION_PLEASE'; const DEFAULT_ADMIN_SECRET = 'admin_setup_secret'; @@ -99,7 +101,12 @@ db.exec(` `); // ── Middleware ──────────────────────────────────────────────────── -app.use(express.json({ limit: '10mb' })); +app.use(express.json({ + limit: '10mb', + verify: (req, _res, buf) => { + req.rawBody = buf; + }, +})); app.use(express.static(path.join(__dirname, 'public'))); function auth(req, res, next) { @@ -127,19 +134,27 @@ function parseMentions(text) { } // Ищем task_id по tasknumber из кэша function taskIdByNumber(taskNumber) { + const row = db.prepare('SELECT task_id FROM tasks_index WHERE task_number=?').get(String(taskNumber)); + if (row?.task_id) return String(row.task_id); const cache = db.prepare('SELECT data FROM tasks_cache WHERE id=1').get(); if (!cache) return null; - const tasks = JSON.parse(cache.data); - const t = tasks.find(t => String(t.tasknumber) === String(taskNumber)); - return t ? String(t.id) : null; + try { + const tasks = JSON.parse(cache.data); + const t = tasks.find((x) => String(x.tasknumber) === String(taskNumber)); + return t ? String(t.id) : null; + } catch { + return null; + } } // Возвращает tasknumber по task_id из кэша function taskNumberById(taskId) { + const row = db.prepare('SELECT task_number FROM tasks_index WHERE task_id=?').get(String(taskId)); + if (row?.task_number) return String(row.task_number); try { const cache = db.prepare('SELECT data FROM tasks_cache WHERE id=1').get(); if (!cache) return taskId; const tasks = JSON.parse(cache.data); - const t = tasks.find(t => String(t.id) === String(taskId)); + const t = tasks.find((x) => String(x.id) === String(taskId)); return t ? String(t.tasknumber || t.id) : taskId; } catch { return taskId; } } @@ -169,6 +184,32 @@ function sseSendTo(u, event, data) { sseClients.get(key)?.forEach(r => { try { r.write(msg); } catch {} }); } +const refreshService = createRefreshService({ + db, + fetch, + getSettings, + sseBroadcast, + logger: console, +}); + +function normalizeSignature(sigValue) { + const raw = String(sigValue || '').trim(); + if (!raw) return ''; + if (raw.startsWith('sha256=')) return raw.slice('sha256='.length).toLowerCase(); + return raw.toLowerCase(); +} + +function verifyWebhookSignature(rawBody, secret, signatureHeader) { + const signature = normalizeSignature(signatureHeader); + if (!signature) return false; + const body = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(rawBody || ''); + const expected = crypto.createHmac('sha256', secret).update(body).digest('hex'); + const expectedBuf = Buffer.from(expected, 'hex'); + const actualBuf = Buffer.from(signature, 'hex'); + if (!expectedBuf.length || expectedBuf.length !== actualBuf.length) return false; + return crypto.timingSafeEqual(expectedBuf, actualBuf); +} + app.get('/healthz', (req, res) => { try { db.prepare('SELECT 1').get(); @@ -247,34 +288,87 @@ app.put('/api/users/:id/password', auth, adminOnly, (req, res) => { app.get('/api/status', auth, (req, res) => { const cache = db.prepare('SELECT updated_at FROM tasks_cache WHERE id=1').get(); const s = getSettings(); + const sync = refreshService.getStatusSnapshot(); res.json({ last_updated: cache?.updated_at || null, auto_refresh_interval: parseInt(s.auto_refresh_interval) || 300, unread_mode: s.unread_mode || 'highlight', export_fields: s.export_fields || '', export_status_mode: s.export_status_mode || 'A', + cache_version: sync.cache_version, + refresh_in_progress: sync.refresh_in_progress, + last_success_at: sync.last_success_at, + last_error: sync.last_error, + last_full_reconcile_at: sync.last_full_reconcile_at, + refresh_mode: refreshService.getMode(), }); }); // ── Tasks ───────────────────────────────────────────────────────── app.get('/api/tasks', auth, (req, res) => { const cache = db.prepare('SELECT * FROM tasks_cache WHERE id=1').get(); - if (!cache) return res.json({ tasks: [], updated_at: null }); - res.json({ tasks: JSON.parse(cache.data), updated_at: cache.updated_at }); + const sync = refreshService.getStatusSnapshot(); + if (!cache) return res.json({ tasks: [], updated_at: null, cache_version: sync.cache_version }); + res.json({ tasks: JSON.parse(cache.data), updated_at: cache.updated_at, cache_version: sync.cache_version }); }); app.post('/api/tasks/refresh', auth, async (req, res) => { - try { res.json({ ok: true, count: await fetchAndCacheTasks(), updated_at: new Date().toISOString() }); } + try { + const result = await refreshService.requestIncremental('manual_api_refresh', { wait: true }); + res.json({ ok: true, ...result }); + } catch (e) { console.error('Refresh error:', e.message); res.status(500).json({ error: e.message }); } }); app.post('/api/tasks/refresh/full', auth, adminOnly, async (req, res) => { try { - const count = await fetchAndCacheAllTasks(); - res.json({ ok: true, count, updated_at: new Date().toISOString() }); + const result = await refreshService.requestFull('manual_api_full_refresh', { wait: true }); + res.json({ ok: true, ...result }); } catch (e) { console.error('Full refresh error:', e.message); res.status(500).json({ error: e.message }); } }); +app.post('/api/webhooks/intradesk', async (req, res) => { + try { + if (refreshService.getMode() !== 'webhook_daily') { + return res.status(202).json({ ok: true, ignored: true, reason: 'refresh_mode_not_webhook_daily' }); + } + + const s = getSettings(); + const webhookSecret = s.webhook_secret || process.env.WEBHOOK_SECRET || ''; + if (!webhookSecret) return res.status(503).json({ error: 'Webhook secret is not configured' }); + + const signature = req.headers['x-webhook-signature'] || req.headers['x-signature'] || ''; + if (!verifyWebhookSignature(req.rawBody, webhookSecret, signature)) { + return res.status(401).json({ error: 'Invalid webhook signature' }); + } + + const body = req.body || {}; + const eventTypeRaw = body.event_type || body.eventType || body.type || body.event || ''; + const { action, eventType } = refreshService.parseWebhookAction(eventTypeRaw); + const taskId = refreshService.extractWebhookTaskId(body); + + if (action === 'incremental') { + await refreshService.requestIncremental(`webhook:${eventType}`, { wait: false }); + return res.json({ ok: true, action: 'incremental_queued' }); + } + + if (action === 'remove') { + if (taskId) { + await refreshService.requestRemoveTask(taskId, `webhook:${eventType}`, { wait: false }); + return res.json({ ok: true, action: 'task_removed_or_queued', task_id: String(taskId) }); + } + await refreshService.requestFull(`webhook:${eventType}:no_task_id`, { wait: false }); + return res.json({ ok: true, action: 'full_reconcile_queued' }); + } + + console.log(`[Webhook] Unknown event type: ${eventTypeRaw}`); + return res.json({ ok: true, ignored: true, reason: 'unknown_event_type', event_type: eventTypeRaw }); + } catch (e) { + console.error('[Webhook] Error:', e.message); + return res.status(500).json({ error: e.message }); + } +}); + // ── Marks ───────────────────────────────────────────────────────── const _marksConfig = require('./public/js/statuses.js'); const ALLOWED_MARKS = new Set(_marksConfig.map(m => m.key)); @@ -443,6 +537,10 @@ const ALLOWED_SETTINGS = new Set([ 'status_ids', 'event_types', 'auto_refresh_interval', + 'refresh_mode', + 'scan_page_size', + 'daily_full_refresh_hour', + 'webhook_secret', 'unread_mode', 'export_fields', 'export_status_mode', @@ -454,17 +552,20 @@ app.get('/api/settings', auth, adminOnly, (req, res) => { status_ids: s.status_ids || '68051,68046', event_types: s.event_types || '', auto_refresh_interval: s.auto_refresh_interval || '300', + refresh_mode: s.refresh_mode || 'incremental_scan', + scan_page_size: s.scan_page_size || '50', + daily_full_refresh_hour: s.daily_full_refresh_hour || '4', unread_mode: s.unread_mode || 'highlight', export_fields: s.export_fields || '', export_status_mode: s.export_status_mode || 'B', api_key_set: !!s.api_key, + webhook_secret_set: !!s.webhook_secret, }); }); app.post('/api/settings', auth, adminOnly, (req, res) => { const body = req.body || {}; Object.entries(body).forEach(([k, v]) => { if (ALLOWED_SETTINGS.has(k) && v !== undefined) setSetting(k, String(v)); }); - if (body.auto_refresh_interval !== undefined) setupAutoRefresh(); res.json({ ok: true }); }); @@ -521,77 +622,8 @@ app.get('/api/tasks/lastcomment/:taskId', auth, async (req, res) => { }); // ── IntraDesk fetch ─────────────────────────────────────────────── -async function buildTasksFromApi(top) { - const s = getSettings(); - if (!s.api_key) throw new Error('API ключ не задан.'); - const executorFilter = [ - 'executorgroup eq 243025', // Поддержка, 1-ая линия - 'executor eq 2227907', // Высоких Дмитрий - 'executor eq 525111', // Абрамов Сергей - 'executor eq 631481', // Железняков Антон - 'executor eq 2222953', // Коваленко Артем - ].join(' or '); - const fullFilter = `(${executorFilter})`; - const topParam = top > 0 ? top : 2000; - const url = `https://apigw.intradesk.ru/tasklist/odata/v3/tasks?ApiKey=${s.api_key}&$filter=${encodeURIComponent(`(${fullFilter})`)}&$orderby=updatedat%20desc&$top=${topParam}`; - console.log(`[IntraDesk] GET top=${topParam}`, url.replace(s.api_key, '***')); - const r = await fetch(url, { timeout: 30000 }); - console.log('[IntraDesk] Status:', r.status, r.statusText); - if (!r.ok) throw new Error(`IntraDesk вернул ${r.status}: ${r.statusText}`); - const json = await r.json(); - const tasks = json.value || []; - const dictMap = {}; - (json.dictionaries || []).forEach(d => { dictMap[d.id] = d.name; }); - tasks.forEach(task => { - if (task.clientid && dictMap[task.clientid]) task._clientname = dictMap[task.clientid]; - if (task.executor && dictMap[task.executor]) task._executorname = dictMap[task.executor]; - if (task.executorgroup && dictMap[task.executorgroup]) task._executorgroupname = dictMap[task.executorgroup]; - // API возвращает статус в поле 'status' (числовой id), иногда 'statusid' — проверяем оба - const statusKey = task.statusid || task.status; - if (statusKey && dictMap[statusKey]) task._statusname = dictMap[statusKey]; - }); - return tasks; -} +refreshService.startScheduler(); -// Quick refresh: fetch top-50 recent tasks and merge into cache -async function fetchAndCacheTasks() { - const fresh = await buildTasksFromApi(50); - const existing = (() => { - try { const c = db.prepare('SELECT data FROM tasks_cache WHERE id=1').get(); return c ? JSON.parse(c.data) : []; } - catch { return []; } - })(); - const byId = new Map(existing.map(t => [t.id, t])); - fresh.forEach(t => byId.set(t.id, t)); - const merged = [...byId.values()].sort((a, b) => (b.updatedat||'') > (a.updatedat||'') ? 1 : -1); - const now = new Date().toISOString(); - db.prepare(`INSERT INTO tasks_cache(id,data,updated_at) VALUES(1,?,?) - ON CONFLICT(id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at`).run(JSON.stringify(merged), now); - sseBroadcast('tasks_updated', { updated_at: now, count: merged.length }); - return merged.length; -} - -// Full refresh: fetch all tasks and replace cache -async function fetchAndCacheAllTasks() { - const tasks = await buildTasksFromApi(0); - const now = new Date().toISOString(); - db.prepare(`INSERT INTO tasks_cache(id,data,updated_at) VALUES(1,?,?) - ON CONFLICT(id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at`).run(JSON.stringify(tasks), now); - sseBroadcast('tasks_updated', { updated_at: now, count: tasks.length }); - return tasks.length; -} - -// ── Auto-refresh ────────────────────────────────────────────────── -let autoRefreshTimer = null; -function setupAutoRefresh() { - if (autoRefreshTimer) { clearInterval(autoRefreshTimer); autoRefreshTimer = null; } - const interval = parseInt(getSettings().auto_refresh_interval) || 0; - if (interval <= 0) { console.log('[AutoRefresh] Выключен'); return; } - autoRefreshTimer = setInterval(async () => { - try { console.log(`[${new Date().toLocaleTimeString('ru-RU')}] AutoRefresh: ${await fetchAndCacheTasks()} заявок`); } - catch (e) { console.error('[AutoRefresh] Ошибка:', e.message); } - }, interval * 1000); - console.log(`[AutoRefresh] Каждые ${interval}с`); -} -setupAutoRefresh(); app.listen(PORT, () => { console.log(`\nServer listening on http://localhost:${PORT}\n`); }); +