progress_monitor.html 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772
  1. <style>
  2. /* Progress Monitor Container */
  3. #progress-monitor {
  4. background: linear-gradient(135deg, #0d1117 0%, #161b22 100%);
  5. color: #c9d1d9;
  6. font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
  7. font-size: 12px;
  8. border-bottom: 1px solid #30363d;
  9. position: relative;
  10. z-index: 100;
  11. }
  12. #progress-monitor.hidden {
  13. display: none;
  14. }
  15. #progress-monitor .tree-container {
  16. max-height: 350px;
  17. overflow-y: auto;
  18. }
  19. /* Header Bar */
  20. #progress-monitor .header-bar {
  21. display: flex;
  22. justify-content: space-between;
  23. align-items: center;
  24. padding: 8px 16px;
  25. background: rgba(0,0,0,0.2);
  26. border-bottom: 1px solid #30363d;
  27. position: sticky;
  28. top: 0;
  29. z-index: 10;
  30. }
  31. #progress-monitor .header-left {
  32. display: flex;
  33. align-items: center;
  34. gap: 16px;
  35. }
  36. #progress-monitor .header-right {
  37. display: flex;
  38. align-items: center;
  39. gap: 12px;
  40. }
  41. /* Orchestrator Status */
  42. #progress-monitor .orchestrator-status {
  43. display: flex;
  44. align-items: center;
  45. gap: 6px;
  46. }
  47. #progress-monitor .status-dot {
  48. width: 8px;
  49. height: 8px;
  50. border-radius: 50%;
  51. flex-shrink: 0;
  52. }
  53. #progress-monitor .status-dot.running {
  54. background: #3fb950;
  55. box-shadow: 0 0 8px #3fb950;
  56. animation: pulse 2s infinite;
  57. }
  58. #progress-monitor .status-dot.idle {
  59. background: #d29922;
  60. box-shadow: 0 0 4px #d29922;
  61. }
  62. #progress-monitor .status-dot.stopped {
  63. background: #6e7681;
  64. }
  65. #progress-monitor .status-dot.flash {
  66. animation: flash 0.3s ease-out;
  67. }
  68. @keyframes pulse {
  69. 0%, 100% { opacity: 1; box-shadow: 0 0 8px #3fb950; }
  70. 50% { opacity: 0.6; box-shadow: 0 0 4px #3fb950; }
  71. }
  72. @keyframes flash {
  73. 0% { transform: scale(1.5); }
  74. 100% { transform: scale(1); }
  75. }
  76. /* Stats */
  77. #progress-monitor .stats {
  78. display: flex;
  79. gap: 16px;
  80. }
  81. #progress-monitor .stat {
  82. display: flex;
  83. align-items: center;
  84. gap: 4px;
  85. }
  86. #progress-monitor .stat-label {
  87. color: #8b949e;
  88. font-size: 10px;
  89. text-transform: uppercase;
  90. letter-spacing: 0.5px;
  91. }
  92. #progress-monitor .stat-value {
  93. font-weight: 600;
  94. font-variant-numeric: tabular-nums;
  95. }
  96. #progress-monitor .stat-value.success { color: #3fb950; }
  97. #progress-monitor .stat-value.error { color: #f85149; }
  98. #progress-monitor .stat-value.warning { color: #d29922; }
  99. #progress-monitor .stat-value.info { color: #58a6ff; }
  100. #progress-monitor .stat.clickable {
  101. cursor: pointer;
  102. padding: 2px 6px;
  103. margin: -2px -6px;
  104. border-radius: 4px;
  105. transition: background 0.2s;
  106. }
  107. #progress-monitor .stat.clickable:hover {
  108. background: rgba(255,255,255,0.1);
  109. }
  110. #progress-monitor .stat.clickable:active {
  111. background: rgba(255,255,255,0.2);
  112. }
  113. /* Toggle Button */
  114. #progress-monitor .toggle-btn {
  115. background: transparent;
  116. border: 1px solid #30363d;
  117. color: #8b949e;
  118. cursor: pointer;
  119. padding: 4px 8px;
  120. border-radius: 6px;
  121. font-size: 11px;
  122. transition: all 0.2s;
  123. }
  124. #progress-monitor .toggle-btn:hover {
  125. background: #21262d;
  126. color: #c9d1d9;
  127. border-color: #8b949e;
  128. }
  129. /* Tree Container */
  130. #progress-monitor .tree-container {
  131. padding: 12px 16px;
  132. }
  133. #progress-monitor.collapsed .tree-container {
  134. display: none;
  135. }
  136. /* Idle Message */
  137. #progress-monitor .idle-message {
  138. color: #8b949e;
  139. font-style: italic;
  140. padding: 8px 0;
  141. text-align: center;
  142. }
  143. /* Crawl Item */
  144. #progress-monitor .crawl-item {
  145. background: #161b22;
  146. border: 1px solid #30363d;
  147. border-radius: 8px;
  148. margin-bottom: 12px;
  149. overflow: hidden;
  150. }
  151. #progress-monitor .crawl-header {
  152. display: flex;
  153. align-items: center;
  154. gap: 12px;
  155. padding: 10px 14px;
  156. background: rgba(0,0,0,0.2);
  157. cursor: pointer;
  158. text-decoration: none;
  159. color: inherit;
  160. }
  161. #progress-monitor .crawl-header:hover {
  162. background: rgba(88, 166, 255, 0.1);
  163. }
  164. #progress-monitor a.crawl-header:visited {
  165. color: inherit;
  166. }
  167. #progress-monitor .crawl-icon {
  168. font-size: 16px;
  169. width: 20px;
  170. text-align: center;
  171. }
  172. #progress-monitor .crawl-info {
  173. flex: 1;
  174. min-width: 0;
  175. }
  176. #progress-monitor .crawl-label {
  177. font-weight: 600;
  178. color: #58a6ff;
  179. white-space: nowrap;
  180. overflow: hidden;
  181. text-overflow: ellipsis;
  182. }
  183. #progress-monitor .crawl-meta {
  184. font-size: 11px;
  185. color: #8b949e;
  186. margin-top: 2px;
  187. }
  188. #progress-monitor .crawl-stats {
  189. display: flex;
  190. gap: 12px;
  191. font-size: 11px;
  192. }
  193. /* Progress Bar */
  194. #progress-monitor .progress-bar-container {
  195. height: 4px;
  196. background: #21262d;
  197. border-radius: 2px;
  198. overflow: hidden;
  199. position: relative;
  200. }
  201. #progress-monitor .progress-bar {
  202. height: 100%;
  203. border-radius: 2px;
  204. transition: width 0.5s ease-out;
  205. position: relative;
  206. }
  207. #progress-monitor .progress-bar.crawl {
  208. background: linear-gradient(90deg, #238636 0%, #3fb950 100%);
  209. }
  210. #progress-monitor .progress-bar.snapshot {
  211. background: linear-gradient(90deg, #1f6feb 0%, #58a6ff 100%);
  212. }
  213. #progress-monitor .progress-bar.extractor {
  214. background: linear-gradient(90deg, #8957e5 0%, #a371f7 100%);
  215. }
  216. #progress-monitor .progress-bar.indeterminate {
  217. background: linear-gradient(90deg, transparent 0%, #58a6ff 50%, transparent 100%);
  218. animation: indeterminate 1.5s infinite linear;
  219. width: 30% !important;
  220. }
  221. @keyframes indeterminate {
  222. 0% { transform: translateX(-100%); }
  223. 100% { transform: translateX(400%); }
  224. }
  225. /* Crawl Body */
  226. #progress-monitor .crawl-body {
  227. padding: 0 14px 14px;
  228. }
  229. #progress-monitor .crawl-progress {
  230. padding: 10px 14px;
  231. border-bottom: 1px solid #21262d;
  232. }
  233. /* Snapshot List */
  234. #progress-monitor .snapshot-list {
  235. margin-top: 8px;
  236. }
  237. #progress-monitor .snapshot-item {
  238. background: #0d1117;
  239. border: 1px solid #21262d;
  240. border-radius: 6px;
  241. margin-bottom: 8px;
  242. overflow: hidden;
  243. }
  244. #progress-monitor .snapshot-header {
  245. display: flex;
  246. align-items: center;
  247. gap: 10px;
  248. padding: 8px 12px;
  249. cursor: pointer;
  250. text-decoration: none;
  251. color: inherit;
  252. }
  253. #progress-monitor .snapshot-header:hover {
  254. background: rgba(88, 166, 255, 0.05);
  255. }
  256. #progress-monitor a.snapshot-header:visited {
  257. color: inherit;
  258. }
  259. #progress-monitor .snapshot-icon {
  260. font-size: 14px;
  261. width: 18px;
  262. text-align: center;
  263. color: #58a6ff;
  264. }
  265. #progress-monitor .snapshot-info {
  266. flex: 1;
  267. min-width: 0;
  268. }
  269. #progress-monitor .snapshot-url {
  270. font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
  271. font-size: 11px;
  272. color: #c9d1d9;
  273. white-space: nowrap;
  274. overflow: hidden;
  275. text-overflow: ellipsis;
  276. }
  277. #progress-monitor .snapshot-meta {
  278. font-size: 10px;
  279. color: #8b949e;
  280. margin-top: 2px;
  281. }
  282. #progress-monitor .snapshot-progress {
  283. padding: 0 12px 8px;
  284. }
  285. /* Extractor List - Compact Badge Layout */
  286. #progress-monitor .extractor-list {
  287. padding: 8px 12px;
  288. background: rgba(0,0,0,0.2);
  289. border-top: 1px solid #21262d;
  290. display: flex;
  291. flex-wrap: wrap;
  292. gap: 4px;
  293. }
  294. #progress-monitor .extractor-badge {
  295. position: relative;
  296. display: inline-flex;
  297. align-items: center;
  298. gap: 4px;
  299. padding: 3px 8px;
  300. border-radius: 4px;
  301. font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
  302. font-size: 10px;
  303. background: #21262d;
  304. overflow: hidden;
  305. white-space: nowrap;
  306. }
  307. #progress-monitor .extractor-badge .progress-fill {
  308. position: absolute;
  309. top: 0;
  310. left: 0;
  311. bottom: 0;
  312. z-index: 0;
  313. transition: width 0.3s ease-out;
  314. }
  315. #progress-monitor .extractor-badge .badge-content {
  316. position: relative;
  317. z-index: 1;
  318. display: flex;
  319. align-items: center;
  320. gap: 4px;
  321. }
  322. #progress-monitor .extractor-badge.queued {
  323. color: #8b949e;
  324. }
  325. #progress-monitor .extractor-badge.queued .progress-fill {
  326. background: rgba(110, 118, 129, 0.2);
  327. width: 0%;
  328. }
  329. #progress-monitor .extractor-badge.started {
  330. color: #d29922;
  331. }
  332. #progress-monitor .extractor-badge.started .progress-fill {
  333. background: rgba(210, 153, 34, 0.3);
  334. width: 50%;
  335. animation: progress-pulse 1.5s ease-in-out infinite;
  336. }
  337. @keyframes progress-pulse {
  338. 0%, 100% { opacity: 0.5; }
  339. 50% { opacity: 1; }
  340. }
  341. #progress-monitor .extractor-badge.succeeded {
  342. color: #3fb950;
  343. }
  344. #progress-monitor .extractor-badge.succeeded .progress-fill {
  345. background: rgba(63, 185, 80, 0.25);
  346. width: 100%;
  347. }
  348. #progress-monitor .extractor-badge.failed {
  349. color: #f85149;
  350. }
  351. #progress-monitor .extractor-badge.failed .progress-fill {
  352. background: rgba(248, 81, 73, 0.25);
  353. width: 100%;
  354. }
  355. #progress-monitor .extractor-badge.backoff {
  356. color: #b8860b;
  357. }
  358. #progress-monitor .extractor-badge.backoff .progress-fill {
  359. background: rgba(210, 153, 34, 0.2);
  360. width: 30%;
  361. }
  362. #progress-monitor .extractor-badge.skipped {
  363. color: #6e7681;
  364. }
  365. #progress-monitor .extractor-badge.skipped .progress-fill {
  366. background: rgba(110, 118, 129, 0.15);
  367. width: 100%;
  368. }
  369. #progress-monitor .extractor-badge .badge-icon {
  370. font-size: 10px;
  371. }
  372. #progress-monitor .extractor-badge.started .badge-icon {
  373. animation: spin 1s linear infinite;
  374. }
  375. @keyframes spin {
  376. from { transform: rotate(0deg); }
  377. to { transform: rotate(360deg); }
  378. }
  379. /* Status Badge */
  380. #progress-monitor .status-badge {
  381. font-size: 10px;
  382. padding: 2px 6px;
  383. border-radius: 10px;
  384. font-weight: 500;
  385. text-transform: uppercase;
  386. letter-spacing: 0.3px;
  387. }
  388. #progress-monitor .status-badge.queued {
  389. background: #21262d;
  390. color: #8b949e;
  391. }
  392. #progress-monitor .status-badge.started {
  393. background: rgba(210, 153, 34, 0.2);
  394. color: #d29922;
  395. }
  396. #progress-monitor .status-badge.sealed,
  397. #progress-monitor .status-badge.succeeded {
  398. background: rgba(63, 185, 80, 0.2);
  399. color: #3fb950;
  400. }
  401. #progress-monitor .status-badge.failed {
  402. background: rgba(248, 81, 73, 0.2);
  403. color: #f85149;
  404. }
  405. #progress-monitor .status-badge.backoff {
  406. background: rgba(210, 153, 34, 0.15);
  407. color: #b8860b;
  408. }
  409. #progress-monitor .status-badge.unknown {
  410. background: #21262d;
  411. color: #6e7681;
  412. }
  413. </style>
  414. <div id="progress-monitor">
  415. <div class="header-bar">
  416. <div class="header-left">
  417. <div class="orchestrator-status">
  418. <span class="status-dot stopped" id="orchestrator-dot"></span>
  419. <span id="orchestrator-text">Stopped</span>
  420. </div>
  421. <div class="stats">
  422. <div class="stat">
  423. <span class="stat-label">Workers</span>
  424. <span class="stat-value info" id="worker-count">0</span>
  425. </div>
  426. <div class="stat">
  427. <span class="stat-label">Queued</span>
  428. <span class="stat-value warning" id="total-queued">0</span>
  429. </div>
  430. <div class="stat clickable" id="stat-succeeded" title="Click to reset counter">
  431. <span class="stat-label">Done</span>
  432. <span class="stat-value success" id="total-succeeded">0</span>
  433. </div>
  434. <div class="stat clickable" id="stat-failed" title="Click to reset counter">
  435. <span class="stat-label">Failed</span>
  436. <span class="stat-value error" id="total-failed">0</span>
  437. </div>
  438. </div>
  439. </div>
  440. <div class="header-right">
  441. <button class="toggle-btn" id="progress-collapse" title="Toggle details">Details</button>
  442. </div>
  443. </div>
  444. <div class="tree-container" id="tree-container">
  445. <div class="idle-message" id="idle-message">No active crawls</div>
  446. <div id="crawl-tree"></div>
  447. </div>
  448. </div>
  449. <script>
  450. (function() {
  451. const monitor = document.getElementById('progress-monitor');
  452. const collapseBtn = document.getElementById('progress-collapse');
  453. const treeContainer = document.getElementById('tree-container');
  454. const crawlTree = document.getElementById('crawl-tree');
  455. const idleMessage = document.getElementById('idle-message');
  456. let pollInterval = null;
  457. let isCollapsed = localStorage.getItem('progress-monitor-collapsed') === 'true';
  458. // Baselines for resettable counters
  459. let succeededBaseline = parseInt(localStorage.getItem('progress-succeeded-baseline') || '0');
  460. let failedBaseline = parseInt(localStorage.getItem('progress-failed-baseline') || '0');
  461. let lastSucceeded = 0;
  462. let lastFailed = 0;
  463. // Click handlers for resetting counters
  464. document.getElementById('stat-succeeded').addEventListener('click', function() {
  465. succeededBaseline = lastSucceeded;
  466. localStorage.setItem('progress-succeeded-baseline', succeededBaseline);
  467. document.getElementById('total-succeeded').textContent = '0';
  468. });
  469. document.getElementById('stat-failed').addEventListener('click', function() {
  470. failedBaseline = lastFailed;
  471. localStorage.setItem('progress-failed-baseline', failedBaseline);
  472. document.getElementById('total-failed').textContent = '0';
  473. });
  474. function formatUrl(url) {
  475. if (!url) return '(no URL)';
  476. try {
  477. const u = new URL(url);
  478. return u.hostname + u.pathname.substring(0, 30) + (u.pathname.length > 30 ? '...' : '');
  479. } catch {
  480. return String(url).substring(0, 50) + (String(url).length > 50 ? '...' : '');
  481. }
  482. }
  483. function renderExtractor(extractor) {
  484. const icon = extractor.status === 'started' ? '&#8635;' :
  485. extractor.status === 'succeeded' ? '&#10003;' :
  486. extractor.status === 'failed' ? '&#10007;' :
  487. extractor.status === 'backoff' ? '&#8987;' :
  488. extractor.status === 'skipped' ? '&#8674;' : '&#9675;';
  489. return `
  490. <span class="extractor-badge ${extractor.status || 'queued'}">
  491. <span class="progress-fill"></span>
  492. <span class="badge-content">
  493. <span class="badge-icon">${icon}</span>
  494. <span>${extractor.plugin || 'unknown'}</span>
  495. </span>
  496. </span>
  497. `;
  498. }
  499. function renderSnapshot(snapshot, crawlId) {
  500. const statusIcon = snapshot.status === 'started' ? '&#8635;' : '&#128196;';
  501. const adminUrl = `/admin/core/snapshot/${snapshot.id || 'unknown'}/change/`;
  502. let extractorHtml = '';
  503. if (snapshot.all_plugins && snapshot.all_plugins.length > 0) {
  504. // Sort plugins alphabetically by name to prevent reordering on updates
  505. const sortedExtractors = [...snapshot.all_plugins].sort((a, b) =>
  506. (a.plugin || '').localeCompare(b.plugin || '')
  507. );
  508. extractorHtml = `
  509. <div class="extractor-list">
  510. ${sortedExtractors.map(e => renderExtractor(e)).join('')}
  511. </div>
  512. `;
  513. }
  514. return `
  515. <div class="snapshot-item">
  516. <a class="snapshot-header" href="${adminUrl}">
  517. <span class="snapshot-icon">${statusIcon}</span>
  518. <div class="snapshot-info">
  519. <div class="snapshot-url">${formatUrl(snapshot.url)}</div>
  520. <div class="snapshot-meta">
  521. ${(snapshot.total_plugins || 0) > 0
  522. ? `${snapshot.completed_plugins || 0}/${snapshot.total_plugins || 0} extractors${(snapshot.failed_plugins || 0) > 0 ? ` <span style="color:#f85149">(${snapshot.failed_plugins} failed)</span>` : ''}`
  523. : 'Waiting for extractors...'}
  524. </div>
  525. </div>
  526. <span class="status-badge ${snapshot.status || 'unknown'}">${snapshot.status || 'unknown'}</span>
  527. </a>
  528. <div class="snapshot-progress">
  529. <div class="progress-bar-container">
  530. <div class="progress-bar snapshot ${snapshot.status === 'started' && (snapshot.progress || 0) === 0 ? 'indeterminate' : ''}"
  531. style="width: ${snapshot.progress || 0}%"></div>
  532. </div>
  533. </div>
  534. ${extractorHtml}
  535. </div>
  536. `;
  537. }
  538. function renderCrawl(crawl) {
  539. const statusIcon = crawl.status === 'started' ? '&#8635;' : '&#128269;';
  540. const adminUrl = `/admin/crawls/crawl/${crawl.id || 'unknown'}/change/`;
  541. let snapshotsHtml = '';
  542. if (crawl.active_snapshots && crawl.active_snapshots.length > 0) {
  543. snapshotsHtml = crawl.active_snapshots.map(s => renderSnapshot(s, crawl.id)).join('');
  544. }
  545. // Show warning if crawl is stuck (queued but can't start)
  546. let warningHtml = '';
  547. if (crawl.status === 'queued' && !crawl.can_start) {
  548. warningHtml = `
  549. <div style="padding: 8px 14px; background: rgba(248, 81, 73, 0.1); border-top: 1px solid #f85149; color: #f85149; font-size: 11px;">
  550. ⚠️ Crawl cannot start: ${crawl.urls_preview ? 'unknown error' : 'no URLs'}
  551. </div>
  552. `;
  553. } else if (crawl.status === 'queued' && crawl.retry_at_future) {
  554. // Queued but retry_at is in future (was claimed by worker, will retry)
  555. warningHtml = `
  556. <div style="padding: 8px 14px; background: rgba(88, 166, 255, 0.1); border-top: 1px solid #58a6ff; color: #58a6ff; font-size: 11px;">
  557. 🔄 Retrying in ${crawl.seconds_until_retry || 0}s...${crawl.urls_preview ? ` (${crawl.urls_preview})` : ''}
  558. </div>
  559. `;
  560. } else if (crawl.status === 'queued' && crawl.total_snapshots === 0) {
  561. // Queued and waiting to be picked up by worker
  562. warningHtml = `
  563. <div style="padding: 8px 14px; background: rgba(210, 153, 34, 0.1); border-top: 1px solid #d29922; color: #d29922; font-size: 11px;">
  564. ⏳ Waiting for worker to pick up...${crawl.urls_preview ? ` (${crawl.urls_preview})` : ''}
  565. </div>
  566. `;
  567. }
  568. // Show snapshot info or URL count if no snapshots yet
  569. let metaText = `depth: ${crawl.max_depth || 0}`;
  570. if ((crawl.total_snapshots || 0) > 0) {
  571. metaText += ` | ${crawl.total_snapshots} snapshots`;
  572. } else if ((crawl.urls_count || 0) > 0) {
  573. metaText += ` | ${crawl.urls_count} URLs`;
  574. } else if (crawl.urls_preview) {
  575. metaText += ` | ${crawl.urls_preview.substring(0, 40)}${crawl.urls_preview.length > 40 ? '...' : ''}`;
  576. }
  577. return `
  578. <div class="crawl-item" data-crawl-id="${crawl.id || 'unknown'}">
  579. <a class="crawl-header" href="${adminUrl}">
  580. <span class="crawl-icon">${statusIcon}</span>
  581. <div class="crawl-info">
  582. <div class="crawl-label">${crawl.label || '(no label)'}</div>
  583. <div class="crawl-meta">${metaText}</div>
  584. </div>
  585. <div class="crawl-stats">
  586. <span style="color:#3fb950">${crawl.completed_snapshots || 0} done</span>
  587. <span style="color:#d29922">${crawl.started_snapshots || 0} active</span>
  588. <span style="color:#8b949e">${crawl.pending_snapshots || 0} pending</span>
  589. </div>
  590. <span class="status-badge ${crawl.status || 'unknown'}">${crawl.status || 'unknown'}</span>
  591. </a>
  592. <div class="crawl-progress">
  593. <div class="progress-bar-container">
  594. <div class="progress-bar crawl ${crawl.status === 'started' && (crawl.progress || 0) === 0 ? 'indeterminate' : ''}"
  595. style="width: ${crawl.progress || 0}%"></div>
  596. </div>
  597. </div>
  598. ${warningHtml}
  599. <div class="crawl-body">
  600. <div class="snapshot-list">
  601. ${snapshotsHtml}
  602. </div>
  603. </div>
  604. </div>
  605. `;
  606. }
  607. function updateProgress(data) {
  608. // Calculate if there's activity
  609. const hasActivity = data.active_crawls.length > 0 ||
  610. data.crawls_pending > 0 || data.crawls_started > 0 ||
  611. data.snapshots_pending > 0 || data.snapshots_started > 0 ||
  612. data.archiveresults_pending > 0 || data.archiveresults_started > 0;
  613. // Update orchestrator status - show "Running" only when there's actual activity
  614. // Don't distinguish between "Stopped" and "Idle" since orchestrator starts/stops frequently
  615. const dot = document.getElementById('orchestrator-dot');
  616. const text = document.getElementById('orchestrator-text');
  617. const hasWorkers = data.total_workers > 0;
  618. if (hasWorkers || hasActivity) {
  619. dot.classList.remove('stopped', 'idle');
  620. dot.classList.add('running');
  621. text.textContent = 'Running';
  622. } else {
  623. // No activity - show as idle (whether orchestrator process exists or not)
  624. dot.classList.remove('stopped', 'running');
  625. dot.classList.add('idle');
  626. text.textContent = 'Idle';
  627. }
  628. // Pulse the dot to show we got fresh data
  629. dot.classList.add('flash');
  630. setTimeout(() => dot.classList.remove('flash'), 300);
  631. // Update stats
  632. document.getElementById('worker-count').textContent = data.total_workers;
  633. document.getElementById('total-queued').textContent =
  634. data.crawls_pending + data.snapshots_pending + data.archiveresults_pending;
  635. // Store raw values and display relative to baseline
  636. lastSucceeded = data.archiveresults_succeeded;
  637. lastFailed = data.archiveresults_failed;
  638. // If baseline is higher than current (e.g. after DB reset), reset baseline
  639. if (succeededBaseline > lastSucceeded) {
  640. succeededBaseline = 0;
  641. localStorage.setItem('progress-succeeded-baseline', '0');
  642. }
  643. if (failedBaseline > lastFailed) {
  644. failedBaseline = 0;
  645. localStorage.setItem('progress-failed-baseline', '0');
  646. }
  647. document.getElementById('total-succeeded').textContent = lastSucceeded - succeededBaseline;
  648. document.getElementById('total-failed').textContent = lastFailed - failedBaseline;
  649. // Render crawl tree
  650. if (data.active_crawls.length > 0) {
  651. idleMessage.style.display = 'none';
  652. crawlTree.innerHTML = data.active_crawls.map(c => renderCrawl(c)).join('');
  653. } else if (hasActivity) {
  654. idleMessage.style.display = 'none';
  655. crawlTree.innerHTML = `
  656. <div class="idle-message">
  657. ${data.snapshots_started || 0} snapshots processing, ${data.archiveresults_started || 0} extractors running
  658. </div>
  659. `;
  660. } else {
  661. idleMessage.style.display = '';
  662. // Build the URL for recent crawls (last 24 hours)
  663. var yesterday = new Date(Date.now() - 24*60*60*1000).toISOString().split('T')[0];
  664. var recentUrl = '/admin/crawls/crawl/?created_at__gte=' + yesterday + '&o=-1';
  665. idleMessage.innerHTML = `No active crawls (${data.crawls_pending || 0} pending, ${data.crawls_started || 0} started, <a href="${recentUrl}" style="color: #58a6ff;">${data.crawls_recent || 0} recent</a>)`;
  666. crawlTree.innerHTML = '';
  667. }
  668. }
  669. function fetchProgress() {
  670. fetch('/admin/live-progress/')
  671. .then(response => response.json())
  672. .then(data => {
  673. if (data.error) {
  674. console.error('Progress API error:', data.error, data.traceback);
  675. idleMessage.textContent = 'API Error: ' + data.error;
  676. idleMessage.style.color = '#f85149';
  677. }
  678. updateProgress(data);
  679. })
  680. .catch(error => {
  681. console.error('Progress fetch error:', error);
  682. idleMessage.textContent = 'Fetch Error: ' + error.message;
  683. idleMessage.style.color = '#f85149';
  684. });
  685. }
  686. function startPolling() {
  687. if (pollInterval) return;
  688. fetchProgress();
  689. pollInterval = setInterval(fetchProgress, 1000); // Poll every 1 second
  690. }
  691. function stopPolling() {
  692. if (pollInterval) {
  693. clearInterval(pollInterval);
  694. pollInterval = null;
  695. }
  696. }
  697. // Collapse toggle
  698. collapseBtn.addEventListener('click', function() {
  699. isCollapsed = !isCollapsed;
  700. localStorage.setItem('progress-monitor-collapsed', isCollapsed);
  701. if (isCollapsed) {
  702. monitor.classList.add('collapsed');
  703. collapseBtn.textContent = 'Expand';
  704. } else {
  705. monitor.classList.remove('collapsed');
  706. collapseBtn.textContent = 'Details';
  707. }
  708. });
  709. // Apply initial state
  710. if (isCollapsed) {
  711. monitor.classList.add('collapsed');
  712. collapseBtn.textContent = 'Expand';
  713. }
  714. // Start polling when page loads
  715. startPolling();
  716. // Pause polling when tab is hidden
  717. document.addEventListener('visibilitychange', function() {
  718. if (document.hidden) {
  719. stopPolling();
  720. } else {
  721. startPolling();
  722. }
  723. });
  724. })();
  725. </script>