|
|
@@ -0,0 +1,2638 @@
|
|
|
+__package__ = 'archivebox.core'
|
|
|
+
|
|
|
+from typing import Optional, Dict, Iterable, Any, List, TYPE_CHECKING
|
|
|
+from archivebox.uuid_compat import uuid7
|
|
|
+from datetime import datetime, timedelta
|
|
|
+from django_stubs_ext.db.models import TypedModelMeta
|
|
|
+
|
|
|
+import os
|
|
|
+import json
|
|
|
+from pathlib import Path
|
|
|
+
|
|
|
+from statemachine import State, registry
|
|
|
+
|
|
|
+from django.db import models
|
|
|
+from django.db.models import QuerySet, Value, Case, When, IntegerField
|
|
|
+from django.utils.functional import cached_property
|
|
|
+from django.utils.text import slugify
|
|
|
+from django.utils import timezone
|
|
|
+from django.core.cache import cache
|
|
|
+from django.urls import reverse, reverse_lazy
|
|
|
+from django.contrib import admin
|
|
|
+from django.conf import settings
|
|
|
+
|
|
|
+from archivebox.config import CONSTANTS
|
|
|
+from archivebox.misc.system import get_dir_size, atomic_write
|
|
|
+from archivebox.misc.util import parse_date, base_url, domain as url_domain, to_json, ts_to_date_str, urlencode, htmlencode, urldecode
|
|
|
+from archivebox.misc.hashing import get_dir_info
|
|
|
+from archivebox.hooks import (
|
|
|
+ EXTRACTOR_INDEXING_PRECEDENCE,
|
|
|
+ get_plugins, get_plugin_name, get_plugin_icon,
|
|
|
+ DEFAULT_PLUGIN_ICONS,
|
|
|
+)
|
|
|
+from archivebox.base_models.models import (
|
|
|
+ ModelWithUUID, ModelWithSerializers, ModelWithOutputDir,
|
|
|
+ ModelWithConfig, ModelWithNotes, ModelWithHealthStats,
|
|
|
+ get_or_create_system_user_pk,
|
|
|
+)
|
|
|
+from workers.models import ModelWithStateMachine, BaseStateMachine
|
|
|
+from workers.tasks import bg_archive_snapshot
|
|
|
+from archivebox.crawls.models import Crawl
|
|
|
+from archivebox.machine.models import NetworkInterface, Binary
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+class Tag(ModelWithSerializers):
|
|
|
+ # Keep AutoField for compatibility with main branch migrations
|
|
|
+ # Don't use UUIDField here - requires complex FK transformation
|
|
|
+ id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
|
|
|
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False, related_name='tag_set')
|
|
|
+ created_at = models.DateTimeField(default=timezone.now, db_index=True, null=True)
|
|
|
+ modified_at = models.DateTimeField(auto_now=True)
|
|
|
+ name = models.CharField(unique=True, blank=False, max_length=100)
|
|
|
+ slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
|
|
|
+
|
|
|
+ snapshot_set: models.Manager['Snapshot']
|
|
|
+
|
|
|
+ class Meta(TypedModelMeta):
|
|
|
+ verbose_name = "Tag"
|
|
|
+ verbose_name_plural = "Tags"
|
|
|
+
|
|
|
+ def __str__(self):
|
|
|
+ return self.name
|
|
|
+
|
|
|
+ def save(self, *args, **kwargs):
|
|
|
+ is_new = self._state.adding
|
|
|
+ if is_new:
|
|
|
+ self.slug = slugify(self.name)
|
|
|
+ existing = set(Tag.objects.filter(slug__startswith=self.slug).values_list("slug", flat=True))
|
|
|
+ i = None
|
|
|
+ while True:
|
|
|
+ slug = f"{slugify(self.name)}_{i}" if i else slugify(self.name)
|
|
|
+ if slug not in existing:
|
|
|
+ self.slug = slug
|
|
|
+ break
|
|
|
+ i = (i or 0) + 1
|
|
|
+ super().save(*args, **kwargs)
|
|
|
+
|
|
|
+ if is_new:
|
|
|
+ from archivebox.misc.logging_util import log_worker_event
|
|
|
+ log_worker_event(
|
|
|
+ worker_type='DB',
|
|
|
+ event='Created Tag',
|
|
|
+ indent_level=0,
|
|
|
+ metadata={
|
|
|
+ 'id': self.id,
|
|
|
+ 'name': self.name,
|
|
|
+ 'slug': self.slug,
|
|
|
+ },
|
|
|
+ )
|
|
|
+
|
|
|
+ @property
|
|
|
+ def api_url(self) -> str:
|
|
|
+ return reverse_lazy('api-1:get_tag', args=[self.id])
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def from_jsonl(record: Dict[str, Any], overrides: Dict[str, Any] = None):
|
|
|
+ """
|
|
|
+ Create/update Tag from JSONL record.
|
|
|
+
|
|
|
+ Args:
|
|
|
+ record: JSONL record with 'name' field
|
|
|
+ overrides: Optional dict with 'snapshot' to auto-attach tag
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ Tag instance or None
|
|
|
+ """
|
|
|
+ from archivebox.misc.jsonl import get_or_create_tag
|
|
|
+
|
|
|
+ try:
|
|
|
+ tag = get_or_create_tag(record)
|
|
|
+
|
|
|
+ # Auto-attach to snapshot if in overrides
|
|
|
+ if overrides and 'snapshot' in overrides and tag:
|
|
|
+ overrides['snapshot'].tags.add(tag)
|
|
|
+
|
|
|
+ return tag
|
|
|
+ except ValueError:
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
+class SnapshotTag(models.Model):
|
|
|
+ id = models.AutoField(primary_key=True)
|
|
|
+ snapshot = models.ForeignKey('Snapshot', db_column='snapshot_id', on_delete=models.CASCADE, to_field='id')
|
|
|
+ tag = models.ForeignKey(Tag, db_column='tag_id', on_delete=models.CASCADE, to_field='id')
|
|
|
+
|
|
|
+ class Meta:
|
|
|
+ db_table = 'core_snapshot_tags'
|
|
|
+ unique_together = [('snapshot', 'tag')]
|
|
|
+
|
|
|
+
|
|
|
+class SnapshotQuerySet(models.QuerySet):
|
|
|
+ """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Filtering Methods
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ FILTER_TYPES = {
|
|
|
+ 'exact': lambda pattern: models.Q(url=pattern),
|
|
|
+ 'substring': lambda pattern: models.Q(url__icontains=pattern),
|
|
|
+ 'regex': lambda pattern: models.Q(url__iregex=pattern),
|
|
|
+ 'domain': lambda pattern: models.Q(url__istartswith=f"http://{pattern}") | models.Q(url__istartswith=f"https://{pattern}") | models.Q(url__istartswith=f"ftp://{pattern}"),
|
|
|
+ 'tag': lambda pattern: models.Q(tags__name=pattern),
|
|
|
+ 'timestamp': lambda pattern: models.Q(timestamp=pattern),
|
|
|
+ }
|
|
|
+
|
|
|
+ def filter_by_patterns(self, patterns: List[str], filter_type: str = 'exact') -> 'SnapshotQuerySet':
|
|
|
+ """Filter snapshots by URL patterns using specified filter type"""
|
|
|
+ from archivebox.misc.logging import stderr
|
|
|
+
|
|
|
+ q_filter = models.Q()
|
|
|
+ for pattern in patterns:
|
|
|
+ try:
|
|
|
+ q_filter = q_filter | self.FILTER_TYPES[filter_type](pattern)
|
|
|
+ except KeyError:
|
|
|
+ stderr()
|
|
|
+ stderr(f'[X] Got invalid pattern for --filter-type={filter_type}:', color='red')
|
|
|
+ stderr(f' {pattern}')
|
|
|
+ raise SystemExit(2)
|
|
|
+ return self.filter(q_filter)
|
|
|
+
|
|
|
+ def search(self, patterns: List[str]) -> 'SnapshotQuerySet':
|
|
|
+ """Search snapshots using the configured search backend"""
|
|
|
+ from archivebox.config.common import SEARCH_BACKEND_CONFIG
|
|
|
+ from archivebox.search import query_search_index
|
|
|
+ from archivebox.misc.logging import stderr
|
|
|
+
|
|
|
+ if not SEARCH_BACKEND_CONFIG.USE_SEARCHING_BACKEND:
|
|
|
+ stderr()
|
|
|
+ stderr('[X] The search backend is not enabled, set config.USE_SEARCHING_BACKEND = True', color='red')
|
|
|
+ raise SystemExit(2)
|
|
|
+
|
|
|
+ qsearch = self.none()
|
|
|
+ for pattern in patterns:
|
|
|
+ try:
|
|
|
+ qsearch |= query_search_index(pattern)
|
|
|
+ except:
|
|
|
+ raise SystemExit(2)
|
|
|
+ return self.all() & qsearch
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Export Methods
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ def to_json(self, with_headers: bool = False) -> str:
|
|
|
+ """Generate JSON index from snapshots"""
|
|
|
+ import sys
|
|
|
+ from datetime import datetime, timezone as tz
|
|
|
+ from archivebox.config import VERSION
|
|
|
+ from archivebox.config.common import SERVER_CONFIG
|
|
|
+
|
|
|
+ MAIN_INDEX_HEADER = {
|
|
|
+ 'info': 'This is an index of site data archived by ArchiveBox: The self-hosted web archive.',
|
|
|
+ 'schema': 'archivebox.index.json',
|
|
|
+ 'copyright_info': SERVER_CONFIG.FOOTER_INFO,
|
|
|
+ 'meta': {
|
|
|
+ 'project': 'ArchiveBox',
|
|
|
+ 'version': VERSION,
|
|
|
+ 'git_sha': VERSION,
|
|
|
+ 'website': 'https://ArchiveBox.io',
|
|
|
+ 'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
|
|
|
+ 'source': 'https://github.com/ArchiveBox/ArchiveBox',
|
|
|
+ 'issues': 'https://github.com/ArchiveBox/ArchiveBox/issues',
|
|
|
+ 'dependencies': {},
|
|
|
+ },
|
|
|
+ } if with_headers else {}
|
|
|
+
|
|
|
+ snapshot_dicts = [s.to_dict(extended=True) for s in self.iterator(chunk_size=500)]
|
|
|
+
|
|
|
+ if with_headers:
|
|
|
+ output = {
|
|
|
+ **MAIN_INDEX_HEADER,
|
|
|
+ 'num_links': len(snapshot_dicts),
|
|
|
+ 'updated': datetime.now(tz.utc),
|
|
|
+ 'last_run_cmd': sys.argv,
|
|
|
+ 'links': snapshot_dicts,
|
|
|
+ }
|
|
|
+ else:
|
|
|
+ output = snapshot_dicts
|
|
|
+ return to_json(output, indent=4, sort_keys=True)
|
|
|
+
|
|
|
+ def to_csv(self, cols: Optional[List[str]] = None, header: bool = True, separator: str = ',', ljust: int = 0) -> str:
|
|
|
+ """Generate CSV output from snapshots"""
|
|
|
+ cols = cols or ['timestamp', 'is_archived', 'url']
|
|
|
+ header_str = separator.join(col.ljust(ljust) for col in cols) if header else ''
|
|
|
+ row_strs = (s.to_csv(cols=cols, ljust=ljust, separator=separator) for s in self.iterator(chunk_size=500))
|
|
|
+ return '\n'.join((header_str, *row_strs))
|
|
|
+
|
|
|
+ def to_html(self, with_headers: bool = True) -> str:
|
|
|
+ """Generate main index HTML from snapshots"""
|
|
|
+ from datetime import datetime, timezone as tz
|
|
|
+ from django.template.loader import render_to_string
|
|
|
+ from archivebox.config import VERSION
|
|
|
+ from archivebox.config.common import SERVER_CONFIG
|
|
|
+ from archivebox.config.version import get_COMMIT_HASH
|
|
|
+
|
|
|
+ template = 'static_index.html' if with_headers else 'minimal_index.html'
|
|
|
+ snapshot_list = list(self.iterator(chunk_size=500))
|
|
|
+
|
|
|
+ return render_to_string(template, {
|
|
|
+ 'version': VERSION,
|
|
|
+ 'git_sha': get_COMMIT_HASH() or VERSION,
|
|
|
+ 'num_links': str(len(snapshot_list)),
|
|
|
+ 'date_updated': datetime.now(tz.utc).strftime('%Y-%m-%d'),
|
|
|
+ 'time_updated': datetime.now(tz.utc).strftime('%Y-%m-%d %H:%M'),
|
|
|
+ 'links': snapshot_list,
|
|
|
+ 'FOOTER_INFO': SERVER_CONFIG.FOOTER_INFO,
|
|
|
+ })
|
|
|
+
|
|
|
+
|
|
|
+class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):
|
|
|
+ """Manager for Snapshot model - uses SnapshotQuerySet for chainable methods"""
|
|
|
+
|
|
|
+ def filter(self, *args, **kwargs):
|
|
|
+ domain = kwargs.pop('domain', None)
|
|
|
+ qs = super().filter(*args, **kwargs)
|
|
|
+ if domain:
|
|
|
+ qs = qs.filter(url__icontains=f'://{domain}')
|
|
|
+ return qs
|
|
|
+
|
|
|
+ def get_queryset(self):
|
|
|
+ return super().get_queryset().prefetch_related('tags', 'archiveresult_set')
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Import Methods
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ def remove(self, atomic: bool = False) -> tuple:
|
|
|
+ """Remove snapshots from the database"""
|
|
|
+ from django.db import transaction
|
|
|
+ if atomic:
|
|
|
+ with transaction.atomic():
|
|
|
+ return self.delete()
|
|
|
+ return self.delete()
|
|
|
+
|
|
|
+
|
|
|
+class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithStateMachine):
|
|
|
+ id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
|
|
+ created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
|
|
+ modified_at = models.DateTimeField(auto_now=True)
|
|
|
+
|
|
|
+ url = models.URLField(unique=False, db_index=True) # URLs can appear in multiple crawls
|
|
|
+ timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
|
|
|
+ bookmarked_at = models.DateTimeField(default=timezone.now, db_index=True)
|
|
|
+ crawl: Crawl = models.ForeignKey(Crawl, on_delete=models.CASCADE, null=False, related_name='snapshot_set', db_index=True) # type: ignore[assignment]
|
|
|
+ parent_snapshot = models.ForeignKey('self', on_delete=models.SET_NULL, null=True, blank=True, related_name='child_snapshots', db_index=True, help_text='Parent snapshot that discovered this URL (for recursive crawling)')
|
|
|
+
|
|
|
+ title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
|
|
|
+ downloaded_at = models.DateTimeField(default=None, null=True, editable=False, db_index=True, blank=True)
|
|
|
+ depth = models.PositiveSmallIntegerField(default=0, db_index=True) # 0 for root snapshot, 1+ for discovered URLs
|
|
|
+ fs_version = models.CharField(max_length=10, default='0.9.0', help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.0"). Used to trigger lazy migration on save().')
|
|
|
+ current_step = models.PositiveSmallIntegerField(default=0, db_index=True, help_text='Current hook step being executed (0-9). Used for sequential hook execution.')
|
|
|
+
|
|
|
+ retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
|
|
|
+ status = ModelWithStateMachine.StatusField(choices=ModelWithStateMachine.StatusChoices, default=ModelWithStateMachine.StatusChoices.QUEUED)
|
|
|
+ config = models.JSONField(default=dict, null=False, blank=False, editable=True)
|
|
|
+ notes = models.TextField(blank=True, null=False, default='')
|
|
|
+ output_dir = models.FilePathField(path=CONSTANTS.ARCHIVE_DIR, recursive=True, match='.*', default=None, null=True, blank=True, editable=True)
|
|
|
+
|
|
|
+ tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
|
|
|
+
|
|
|
+ state_machine_name = 'core.models.SnapshotMachine'
|
|
|
+ state_field_name = 'status'
|
|
|
+ retry_at_field_name = 'retry_at'
|
|
|
+ StatusChoices = ModelWithStateMachine.StatusChoices
|
|
|
+ active_state = StatusChoices.STARTED
|
|
|
+
|
|
|
+ objects = SnapshotManager()
|
|
|
+ archiveresult_set: models.Manager['ArchiveResult']
|
|
|
+
|
|
|
+ class Meta(TypedModelMeta):
|
|
|
+ verbose_name = "Snapshot"
|
|
|
+ verbose_name_plural = "Snapshots"
|
|
|
+ constraints = [
|
|
|
+ # Allow same URL in different crawls, but not duplicates within same crawl
|
|
|
+ models.UniqueConstraint(fields=['url', 'crawl'], name='unique_url_per_crawl'),
|
|
|
+ # Global timestamp uniqueness for 1:1 symlink mapping
|
|
|
+ models.UniqueConstraint(fields=['timestamp'], name='unique_timestamp'),
|
|
|
+ ]
|
|
|
+
|
|
|
+ def __str__(self):
|
|
|
+ return f'[{self.id}] {self.url[:64]}'
|
|
|
+
|
|
|
+ def save(self, *args, **kwargs):
|
|
|
+ is_new = self._state.adding
|
|
|
+ if not self.bookmarked_at:
|
|
|
+ self.bookmarked_at = self.created_at or timezone.now()
|
|
|
+ if not self.timestamp:
|
|
|
+ self.timestamp = str(self.bookmarked_at.timestamp())
|
|
|
+
|
|
|
+ # Migrate filesystem if needed (happens automatically on save)
|
|
|
+ if self.pk and self.fs_migration_needed:
|
|
|
+ from django.db import transaction
|
|
|
+ with transaction.atomic():
|
|
|
+ # Walk through migration chain automatically
|
|
|
+ current = self.fs_version
|
|
|
+ target = self._fs_current_version()
|
|
|
+
|
|
|
+ while current != target:
|
|
|
+ next_ver = self._fs_next_version(current)
|
|
|
+ method = f'_fs_migrate_from_{current.replace(".", "_")}_to_{next_ver.replace(".", "_")}'
|
|
|
+
|
|
|
+ # Only run if method exists (most are no-ops)
|
|
|
+ if hasattr(self, method):
|
|
|
+ getattr(self, method)()
|
|
|
+
|
|
|
+ current = next_ver
|
|
|
+
|
|
|
+ # Update version (still in transaction)
|
|
|
+ self.fs_version = target
|
|
|
+
|
|
|
+ super().save(*args, **kwargs)
|
|
|
+ if self.crawl and self.url not in self.crawl.urls:
|
|
|
+ self.crawl.urls += f'\n{self.url}'
|
|
|
+ self.crawl.save()
|
|
|
+
|
|
|
+ if is_new:
|
|
|
+ from archivebox.misc.logging_util import log_worker_event
|
|
|
+ log_worker_event(
|
|
|
+ worker_type='DB',
|
|
|
+ event='Created Snapshot',
|
|
|
+ indent_level=2,
|
|
|
+ url=self.url,
|
|
|
+ metadata={
|
|
|
+ 'id': str(self.id),
|
|
|
+ 'crawl_id': str(self.crawl_id) if self.crawl_id else None,
|
|
|
+ 'depth': self.depth,
|
|
|
+ 'status': self.status,
|
|
|
+ },
|
|
|
+ )
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Filesystem Migration Methods
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _fs_current_version() -> str:
|
|
|
+ """Get current ArchiveBox filesystem version (normalized to x.x.0 format)"""
|
|
|
+ from archivebox.config import VERSION
|
|
|
+ # Normalize version to x.x.0 format (e.g., "0.9.0rc1" -> "0.9.0")
|
|
|
+ parts = VERSION.split('.')
|
|
|
+ if len(parts) >= 2:
|
|
|
+ major, minor = parts[0], parts[1]
|
|
|
+ # Strip any non-numeric suffix from minor version
|
|
|
+ minor = ''.join(c for c in minor if c.isdigit())
|
|
|
+ return f'{major}.{minor}.0'
|
|
|
+ return '0.9.0' # Fallback if version parsing fails
|
|
|
+
|
|
|
+ @property
|
|
|
+ def fs_migration_needed(self) -> bool:
|
|
|
+ """Check if snapshot needs filesystem migration"""
|
|
|
+ return self.fs_version != self._fs_current_version()
|
|
|
+
|
|
|
+ def _fs_next_version(self, version: str) -> str:
|
|
|
+ """Get next version in migration chain"""
|
|
|
+ chain = ['0.7.0', '0.8.0', '0.9.0']
|
|
|
+ try:
|
|
|
+ idx = chain.index(version)
|
|
|
+ return chain[idx + 1] if idx + 1 < len(chain) else self._fs_current_version()
|
|
|
+ except ValueError:
|
|
|
+ # Unknown version - skip to current
|
|
|
+ return self._fs_current_version()
|
|
|
+
|
|
|
+ def _fs_migrate_from_0_7_0_to_0_8_0(self):
|
|
|
+ """Migration from 0.7.0 to 0.8.0 layout (no-op)"""
|
|
|
+ # 0.7 and 0.8 both used archive/<timestamp>
|
|
|
+ # Nothing to do!
|
|
|
+ pass
|
|
|
+
|
|
|
+ def _fs_migrate_from_0_8_0_to_0_9_0(self):
|
|
|
+ """
|
|
|
+ Migrate from flat to nested structure.
|
|
|
+
|
|
|
+ 0.8.x: archive/{timestamp}/
|
|
|
+ 0.9.x: users/{user}/snapshots/YYYYMMDD/{domain}/{uuid}/
|
|
|
+
|
|
|
+ Transaction handling:
|
|
|
+ 1. Copy files INSIDE transaction
|
|
|
+ 2. Create symlink INSIDE transaction
|
|
|
+ 3. Update fs_version INSIDE transaction (done by save())
|
|
|
+ 4. Exit transaction (DB commit)
|
|
|
+ 5. Delete old files OUTSIDE transaction (after commit)
|
|
|
+ """
|
|
|
+ import shutil
|
|
|
+ from django.db import transaction
|
|
|
+
|
|
|
+ old_dir = self.get_storage_path_for_version('0.8.0')
|
|
|
+ new_dir = self.get_storage_path_for_version('0.9.0')
|
|
|
+
|
|
|
+ if not old_dir.exists() or old_dir == new_dir or new_dir.exists():
|
|
|
+ return
|
|
|
+
|
|
|
+ new_dir.mkdir(parents=True, exist_ok=True)
|
|
|
+
|
|
|
+ # Copy all files (idempotent)
|
|
|
+ for old_file in old_dir.rglob('*'):
|
|
|
+ if not old_file.is_file():
|
|
|
+ continue
|
|
|
+
|
|
|
+ rel_path = old_file.relative_to(old_dir)
|
|
|
+ new_file = new_dir / rel_path
|
|
|
+
|
|
|
+ # Skip if already copied
|
|
|
+ if new_file.exists() and new_file.stat().st_size == old_file.stat().st_size:
|
|
|
+ continue
|
|
|
+
|
|
|
+ new_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
+ shutil.copy2(old_file, new_file)
|
|
|
+
|
|
|
+ # Verify all copied
|
|
|
+ old_files = {f.relative_to(old_dir): f.stat().st_size
|
|
|
+ for f in old_dir.rglob('*') if f.is_file()}
|
|
|
+ new_files = {f.relative_to(new_dir): f.stat().st_size
|
|
|
+ for f in new_dir.rglob('*') if f.is_file()}
|
|
|
+
|
|
|
+ if old_files.keys() != new_files.keys():
|
|
|
+ missing = old_files.keys() - new_files.keys()
|
|
|
+ raise Exception(f"Migration incomplete: missing {missing}")
|
|
|
+
|
|
|
+ # Create backwards-compat symlink (INSIDE transaction)
|
|
|
+ symlink_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
|
|
|
+ if symlink_path.is_symlink():
|
|
|
+ symlink_path.unlink()
|
|
|
+
|
|
|
+ if not symlink_path.exists() or symlink_path == old_dir:
|
|
|
+ symlink_path.symlink_to(new_dir, target_is_directory=True)
|
|
|
+
|
|
|
+ # Schedule old directory deletion AFTER transaction commits
|
|
|
+ transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir))
|
|
|
+
|
|
|
+ def _cleanup_old_migration_dir(self, old_dir: Path):
|
|
|
+ """
|
|
|
+ Delete old directory after successful migration.
|
|
|
+ Called via transaction.on_commit() after DB commit succeeds.
|
|
|
+ """
|
|
|
+ import shutil
|
|
|
+ import logging
|
|
|
+
|
|
|
+ if old_dir.exists() and not old_dir.is_symlink():
|
|
|
+ try:
|
|
|
+ shutil.rmtree(old_dir)
|
|
|
+ except Exception as e:
|
|
|
+ # Log but don't raise - migration succeeded, this is just cleanup
|
|
|
+ logging.getLogger('archivebox.migration').warning(
|
|
|
+ f"Could not remove old migration directory {old_dir}: {e}"
|
|
|
+ )
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Path Calculation and Migration Helpers
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def extract_domain_from_url(url: str) -> str:
|
|
|
+ """
|
|
|
+ Extract domain from URL for 0.9.x path structure.
|
|
|
+ Uses full hostname with sanitized special chars.
|
|
|
+
|
|
|
+ Examples:
|
|
|
+ https://example.com:8080 → example.com_8080
|
|
|
+ https://sub.example.com → sub.example.com
|
|
|
+ file:///path → localhost
|
|
|
+ data:text/html → data
|
|
|
+ """
|
|
|
+ from urllib.parse import urlparse
|
|
|
+
|
|
|
+ try:
|
|
|
+ parsed = urlparse(url)
|
|
|
+
|
|
|
+ if parsed.scheme in ('http', 'https'):
|
|
|
+ if parsed.port:
|
|
|
+ return f"{parsed.hostname}_{parsed.port}".replace(':', '_')
|
|
|
+ return parsed.hostname or 'unknown'
|
|
|
+ elif parsed.scheme == 'file':
|
|
|
+ return 'localhost'
|
|
|
+ elif parsed.scheme:
|
|
|
+ return parsed.scheme
|
|
|
+ else:
|
|
|
+ return 'unknown'
|
|
|
+ except Exception:
|
|
|
+ return 'unknown'
|
|
|
+
|
|
|
+ def get_storage_path_for_version(self, version: str) -> Path:
|
|
|
+ """
|
|
|
+ Calculate storage path for specific filesystem version.
|
|
|
+ Centralizes path logic so it's reusable.
|
|
|
+
|
|
|
+ 0.7.x/0.8.x: archive/{timestamp}
|
|
|
+ 0.9.x: users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/
|
|
|
+ """
|
|
|
+ from datetime import datetime
|
|
|
+
|
|
|
+ if version in ('0.7.0', '0.8.0'):
|
|
|
+ return CONSTANTS.ARCHIVE_DIR / self.timestamp
|
|
|
+
|
|
|
+ elif version in ('0.9.0', '1.0.0'):
|
|
|
+ username = self.crawl.created_by.username
|
|
|
+
|
|
|
+ # Use created_at for date grouping (fallback to timestamp)
|
|
|
+ if self.created_at:
|
|
|
+ date_str = self.created_at.strftime('%Y%m%d')
|
|
|
+ else:
|
|
|
+ date_str = datetime.fromtimestamp(float(self.timestamp)).strftime('%Y%m%d')
|
|
|
+
|
|
|
+ domain = self.extract_domain_from_url(self.url)
|
|
|
+
|
|
|
+ return (
|
|
|
+ CONSTANTS.DATA_DIR / 'users' / username / 'snapshots' /
|
|
|
+ date_str / domain / str(self.id)
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ # Unknown version - use current
|
|
|
+ return self.get_storage_path_for_version(self._fs_current_version())
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Loading and Creation from Filesystem (Used by archivebox update ONLY)
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def load_from_directory(cls, snapshot_dir: Path) -> Optional['Snapshot']:
|
|
|
+ """
|
|
|
+ Load existing Snapshot from DB by reading index.json.
|
|
|
+
|
|
|
+ Reads index.json, extracts url+timestamp, queries DB.
|
|
|
+ Returns existing Snapshot or None if not found/invalid.
|
|
|
+ Does NOT create new snapshots.
|
|
|
+
|
|
|
+ ONLY used by: archivebox update (for orphan detection)
|
|
|
+ """
|
|
|
+ import json
|
|
|
+
|
|
|
+ index_path = snapshot_dir / 'index.json'
|
|
|
+ if not index_path.exists():
|
|
|
+ return None
|
|
|
+
|
|
|
+ try:
|
|
|
+ with open(index_path) as f:
|
|
|
+ data = json.load(f)
|
|
|
+ except:
|
|
|
+ return None
|
|
|
+
|
|
|
+ url = data.get('url')
|
|
|
+ if not url:
|
|
|
+ return None
|
|
|
+
|
|
|
+ # Get timestamp - prefer index.json, fallback to folder name
|
|
|
+ timestamp = cls._select_best_timestamp(
|
|
|
+ index_timestamp=data.get('timestamp'),
|
|
|
+ folder_name=snapshot_dir.name
|
|
|
+ )
|
|
|
+
|
|
|
+ if not timestamp:
|
|
|
+ return None
|
|
|
+
|
|
|
+ # Look up existing
|
|
|
+ try:
|
|
|
+ return cls.objects.get(url=url, timestamp=timestamp)
|
|
|
+ except cls.DoesNotExist:
|
|
|
+ return None
|
|
|
+ except cls.MultipleObjectsReturned:
|
|
|
+ # Should not happen with unique constraint
|
|
|
+ return cls.objects.filter(url=url, timestamp=timestamp).first()
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def create_from_directory(cls, snapshot_dir: Path) -> Optional['Snapshot']:
|
|
|
+ """
|
|
|
+ Create new Snapshot from orphaned directory.
|
|
|
+
|
|
|
+ Validates timestamp, ensures uniqueness.
|
|
|
+ Returns new UNSAVED Snapshot or None if invalid.
|
|
|
+
|
|
|
+ ONLY used by: archivebox update (for orphan import)
|
|
|
+ """
|
|
|
+ import json
|
|
|
+
|
|
|
+ index_path = snapshot_dir / 'index.json'
|
|
|
+ if not index_path.exists():
|
|
|
+ return None
|
|
|
+
|
|
|
+ try:
|
|
|
+ with open(index_path) as f:
|
|
|
+ data = json.load(f)
|
|
|
+ except:
|
|
|
+ return None
|
|
|
+
|
|
|
+ url = data.get('url')
|
|
|
+ if not url:
|
|
|
+ return None
|
|
|
+
|
|
|
+ # Get and validate timestamp
|
|
|
+ timestamp = cls._select_best_timestamp(
|
|
|
+ index_timestamp=data.get('timestamp'),
|
|
|
+ folder_name=snapshot_dir.name
|
|
|
+ )
|
|
|
+
|
|
|
+ if not timestamp:
|
|
|
+ return None
|
|
|
+
|
|
|
+ # Ensure uniqueness (reuses existing logic from create_or_update_from_dict)
|
|
|
+ timestamp = cls._ensure_unique_timestamp(url, timestamp)
|
|
|
+
|
|
|
+ # Detect version
|
|
|
+ fs_version = cls._detect_fs_version_from_index(data)
|
|
|
+
|
|
|
+ return cls(
|
|
|
+ url=url,
|
|
|
+ timestamp=timestamp,
|
|
|
+ title=data.get('title', ''),
|
|
|
+ fs_version=fs_version,
|
|
|
+ created_by_id=get_or_create_system_user_pk(),
|
|
|
+ )
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _select_best_timestamp(index_timestamp: str, folder_name: str) -> Optional[str]:
|
|
|
+ """
|
|
|
+ Select best timestamp from index.json vs folder name.
|
|
|
+
|
|
|
+ Validates range (1995-2035).
|
|
|
+ Prefers index.json if valid.
|
|
|
+ """
|
|
|
+ def is_valid_timestamp(ts):
|
|
|
+ try:
|
|
|
+ ts_int = int(float(ts))
|
|
|
+ # 1995-01-01 to 2035-12-31
|
|
|
+ return 788918400 <= ts_int <= 2082758400
|
|
|
+ except:
|
|
|
+ return False
|
|
|
+
|
|
|
+ index_valid = is_valid_timestamp(index_timestamp) if index_timestamp else False
|
|
|
+ folder_valid = is_valid_timestamp(folder_name)
|
|
|
+
|
|
|
+ if index_valid:
|
|
|
+ return str(int(float(index_timestamp)))
|
|
|
+ elif folder_valid:
|
|
|
+ return str(int(float(folder_name)))
|
|
|
+ else:
|
|
|
+ return None
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def _ensure_unique_timestamp(cls, url: str, timestamp: str) -> str:
|
|
|
+ """
|
|
|
+ Ensure timestamp is globally unique.
|
|
|
+ If collision with different URL, increment by 1 until unique.
|
|
|
+
|
|
|
+ NOTE: Logic already exists in create_or_update_from_dict (line 266-267)
|
|
|
+ This is just an extracted, reusable version.
|
|
|
+ """
|
|
|
+ while cls.objects.filter(timestamp=timestamp).exclude(url=url).exists():
|
|
|
+ timestamp = str(int(float(timestamp)) + 1)
|
|
|
+ return timestamp
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _detect_fs_version_from_index(data: dict) -> str:
|
|
|
+ """
|
|
|
+ Detect fs_version from index.json structure.
|
|
|
+
|
|
|
+ - Has fs_version field: use it
|
|
|
+ - Has history dict: 0.7.0
|
|
|
+ - Has archive_results list: 0.8.0
|
|
|
+ - Default: 0.7.0
|
|
|
+ """
|
|
|
+ if 'fs_version' in data:
|
|
|
+ return data['fs_version']
|
|
|
+ if 'history' in data and 'archive_results' not in data:
|
|
|
+ return '0.7.0'
|
|
|
+ if 'archive_results' in data:
|
|
|
+ return '0.8.0'
|
|
|
+ return '0.7.0'
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Index.json Reconciliation
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ def reconcile_with_index_json(self):
|
|
|
+ """
|
|
|
+ Merge index.json with DB. DB is source of truth.
|
|
|
+
|
|
|
+ - Title: longest non-URL
|
|
|
+ - Tags: union
|
|
|
+ - ArchiveResults: keep both (by plugin+start_ts)
|
|
|
+
|
|
|
+ Writes back in 0.9.x format.
|
|
|
+
|
|
|
+ Used by: archivebox update (to sync index.json with DB)
|
|
|
+ """
|
|
|
+ import json
|
|
|
+
|
|
|
+ index_path = Path(self.output_dir) / 'index.json'
|
|
|
+
|
|
|
+ index_data = {}
|
|
|
+ if index_path.exists():
|
|
|
+ try:
|
|
|
+ with open(index_path) as f:
|
|
|
+ index_data = json.load(f)
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ # Merge title
|
|
|
+ self._merge_title_from_index(index_data)
|
|
|
+
|
|
|
+ # Merge tags
|
|
|
+ self._merge_tags_from_index(index_data)
|
|
|
+
|
|
|
+ # Merge ArchiveResults
|
|
|
+ self._merge_archive_results_from_index(index_data)
|
|
|
+
|
|
|
+ # Write back
|
|
|
+ self.write_index_json()
|
|
|
+
|
|
|
+ def _merge_title_from_index(self, index_data: dict):
|
|
|
+ """Merge title - prefer longest non-URL title."""
|
|
|
+ index_title = index_data.get('title', '').strip()
|
|
|
+ db_title = self.title or ''
|
|
|
+
|
|
|
+ candidates = [t for t in [index_title, db_title] if t and t != self.url]
|
|
|
+ if candidates:
|
|
|
+ best_title = max(candidates, key=len)
|
|
|
+ if self.title != best_title:
|
|
|
+ self.title = best_title
|
|
|
+
|
|
|
+ def _merge_tags_from_index(self, index_data: dict):
|
|
|
+ """Merge tags - union of both sources."""
|
|
|
+ from django.db import transaction
|
|
|
+
|
|
|
+ index_tags = set(index_data.get('tags', '').split(',')) if index_data.get('tags') else set()
|
|
|
+ index_tags = {t.strip() for t in index_tags if t.strip()}
|
|
|
+
|
|
|
+ db_tags = set(self.tags.values_list('name', flat=True))
|
|
|
+
|
|
|
+ new_tags = index_tags - db_tags
|
|
|
+ if new_tags:
|
|
|
+ with transaction.atomic():
|
|
|
+ for tag_name in new_tags:
|
|
|
+ tag, _ = Tag.objects.get_or_create(name=tag_name)
|
|
|
+ self.tags.add(tag)
|
|
|
+
|
|
|
+ def _merge_archive_results_from_index(self, index_data: dict):
|
|
|
+ """Merge ArchiveResults - keep both (by plugin+start_ts)."""
|
|
|
+ existing = {
|
|
|
+ (ar.plugin, ar.start_ts): ar
|
|
|
+ for ar in ArchiveResult.objects.filter(snapshot=self)
|
|
|
+ }
|
|
|
+
|
|
|
+ # Handle 0.8.x format (archive_results list)
|
|
|
+ for result_data in index_data.get('archive_results', []):
|
|
|
+ self._create_archive_result_if_missing(result_data, existing)
|
|
|
+
|
|
|
+ # Handle 0.7.x format (history dict)
|
|
|
+ if 'history' in index_data and isinstance(index_data['history'], dict):
|
|
|
+ for plugin, result_list in index_data['history'].items():
|
|
|
+ if isinstance(result_list, list):
|
|
|
+ for result_data in result_list:
|
|
|
+ # Support both old 'extractor' and new 'plugin' keys for backwards compat
|
|
|
+ result_data['plugin'] = result_data.get('plugin') or result_data.get('extractor') or plugin
|
|
|
+ self._create_archive_result_if_missing(result_data, existing)
|
|
|
+
|
|
|
+ def _create_archive_result_if_missing(self, result_data: dict, existing: dict):
|
|
|
+ """Create ArchiveResult if not already in DB."""
|
|
|
+ from dateutil import parser
|
|
|
+
|
|
|
+ # Support both old 'extractor' and new 'plugin' keys for backwards compat
|
|
|
+ plugin = result_data.get('plugin') or result_data.get('extractor', '')
|
|
|
+ if not plugin:
|
|
|
+ return
|
|
|
+
|
|
|
+ start_ts = None
|
|
|
+ if result_data.get('start_ts'):
|
|
|
+ try:
|
|
|
+ start_ts = parser.parse(result_data['start_ts'])
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ if (plugin, start_ts) in existing:
|
|
|
+ return
|
|
|
+
|
|
|
+ try:
|
|
|
+ end_ts = None
|
|
|
+ if result_data.get('end_ts'):
|
|
|
+ try:
|
|
|
+ end_ts = parser.parse(result_data['end_ts'])
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ ArchiveResult.objects.create(
|
|
|
+ snapshot=self,
|
|
|
+ plugin=plugin,
|
|
|
+ hook_name=result_data.get('hook_name', ''),
|
|
|
+ status=result_data.get('status', 'failed'),
|
|
|
+ output_str=result_data.get('output', ''),
|
|
|
+ cmd=result_data.get('cmd', []),
|
|
|
+ pwd=result_data.get('pwd', str(self.output_dir)),
|
|
|
+ start_ts=start_ts,
|
|
|
+ end_ts=end_ts,
|
|
|
+ created_by=self.crawl.created_by,
|
|
|
+ )
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ def write_index_json(self):
|
|
|
+ """Write index.json in 0.9.x format."""
|
|
|
+ import json
|
|
|
+
|
|
|
+ index_path = Path(self.output_dir) / 'index.json'
|
|
|
+
|
|
|
+ data = {
|
|
|
+ 'url': self.url,
|
|
|
+ 'timestamp': self.timestamp,
|
|
|
+ 'title': self.title or '',
|
|
|
+ 'tags': ','.join(sorted(self.tags.values_list('name', flat=True))),
|
|
|
+ 'fs_version': self.fs_version,
|
|
|
+ 'bookmarked_at': self.bookmarked_at.isoformat() if self.bookmarked_at else None,
|
|
|
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
|
|
|
+ 'archive_results': [
|
|
|
+ {
|
|
|
+ 'plugin': ar.plugin,
|
|
|
+ 'status': ar.status,
|
|
|
+ 'start_ts': ar.start_ts.isoformat() if ar.start_ts else None,
|
|
|
+ 'end_ts': ar.end_ts.isoformat() if ar.end_ts else None,
|
|
|
+ 'output': ar.output_str or '',
|
|
|
+ 'cmd': ar.cmd if isinstance(ar.cmd, list) else [],
|
|
|
+ 'pwd': ar.pwd,
|
|
|
+ }
|
|
|
+ for ar in ArchiveResult.objects.filter(snapshot=self).order_by('start_ts')
|
|
|
+ ],
|
|
|
+ }
|
|
|
+
|
|
|
+ index_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
+ with open(index_path, 'w') as f:
|
|
|
+ json.dump(data, f, indent=2, sort_keys=True)
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Snapshot Utilities
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def move_directory_to_invalid(snapshot_dir: Path):
|
|
|
+ """
|
|
|
+ Move invalid directory to data/invalid/YYYYMMDD/.
|
|
|
+
|
|
|
+ Used by: archivebox update (when encountering invalid directories)
|
|
|
+ """
|
|
|
+ from datetime import datetime
|
|
|
+ import shutil
|
|
|
+
|
|
|
+ invalid_dir = CONSTANTS.DATA_DIR / 'invalid' / datetime.now().strftime('%Y%m%d')
|
|
|
+ invalid_dir.mkdir(parents=True, exist_ok=True)
|
|
|
+
|
|
|
+ dest = invalid_dir / snapshot_dir.name
|
|
|
+ counter = 1
|
|
|
+ while dest.exists():
|
|
|
+ dest = invalid_dir / f"{snapshot_dir.name}_{counter}"
|
|
|
+ counter += 1
|
|
|
+
|
|
|
+ try:
|
|
|
+ shutil.move(str(snapshot_dir), str(dest))
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def find_and_merge_duplicates(cls) -> int:
|
|
|
+ """
|
|
|
+ Find and merge snapshots with same url:timestamp.
|
|
|
+ Returns count of duplicate sets merged.
|
|
|
+
|
|
|
+ Used by: archivebox update (Phase 3: deduplication)
|
|
|
+ """
|
|
|
+ from django.db.models import Count
|
|
|
+
|
|
|
+ duplicates = (
|
|
|
+ cls.objects
|
|
|
+ .values('url', 'timestamp')
|
|
|
+ .annotate(count=Count('id'))
|
|
|
+ .filter(count__gt=1)
|
|
|
+ )
|
|
|
+
|
|
|
+ merged = 0
|
|
|
+ for dup in duplicates.iterator():
|
|
|
+ snapshots = list(
|
|
|
+ cls.objects
|
|
|
+ .filter(url=dup['url'], timestamp=dup['timestamp'])
|
|
|
+ .order_by('created_at') # Keep oldest
|
|
|
+ )
|
|
|
+
|
|
|
+ if len(snapshots) > 1:
|
|
|
+ try:
|
|
|
+ cls._merge_snapshots(snapshots)
|
|
|
+ merged += 1
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ return merged
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def _merge_snapshots(cls, snapshots: list['Snapshot']):
|
|
|
+ """
|
|
|
+ Merge exact duplicates.
|
|
|
+ Keep oldest, union files + ArchiveResults.
|
|
|
+ """
|
|
|
+ import shutil
|
|
|
+
|
|
|
+ keeper = snapshots[0]
|
|
|
+ duplicates = snapshots[1:]
|
|
|
+
|
|
|
+ keeper_dir = Path(keeper.output_dir)
|
|
|
+
|
|
|
+ for dup in duplicates:
|
|
|
+ dup_dir = Path(dup.output_dir)
|
|
|
+
|
|
|
+ # Merge files
|
|
|
+ if dup_dir.exists() and dup_dir != keeper_dir:
|
|
|
+ for dup_file in dup_dir.rglob('*'):
|
|
|
+ if not dup_file.is_file():
|
|
|
+ continue
|
|
|
+
|
|
|
+ rel = dup_file.relative_to(dup_dir)
|
|
|
+ keeper_file = keeper_dir / rel
|
|
|
+
|
|
|
+ if not keeper_file.exists():
|
|
|
+ keeper_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
+ shutil.copy2(dup_file, keeper_file)
|
|
|
+
|
|
|
+ try:
|
|
|
+ shutil.rmtree(dup_dir)
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ # Merge tags
|
|
|
+ for tag in dup.tags.all():
|
|
|
+ keeper.tags.add(tag)
|
|
|
+
|
|
|
+ # Move ArchiveResults
|
|
|
+ ArchiveResult.objects.filter(snapshot=dup).update(snapshot=keeper)
|
|
|
+
|
|
|
+ # Delete
|
|
|
+ dup.delete()
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Output Directory Properties
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @property
|
|
|
+ def output_dir_parent(self) -> str:
|
|
|
+ return 'archive'
|
|
|
+
|
|
|
+ @property
|
|
|
+ def output_dir_name(self) -> str:
|
|
|
+ return str(self.timestamp)
|
|
|
+
|
|
|
+ def archive(self, overwrite=False, methods=None):
|
|
|
+ return bg_archive_snapshot(self, overwrite=overwrite, methods=methods)
|
|
|
+
|
|
|
+ @admin.display(description='Tags')
|
|
|
+ def tags_str(self, nocache=True) -> str | None:
|
|
|
+ calc_tags_str = lambda: ','.join(sorted(tag.name for tag in self.tags.all()))
|
|
|
+ if hasattr(self, '_prefetched_objects_cache') and 'tags' in self._prefetched_objects_cache:
|
|
|
+ return calc_tags_str()
|
|
|
+ cache_key = f'{self.pk}-tags'
|
|
|
+ return cache.get_or_set(cache_key, calc_tags_str) if not nocache else calc_tags_str()
|
|
|
+
|
|
|
+ def icons(self) -> str:
|
|
|
+ """Generate HTML icons showing which extractor plugins have succeeded for this snapshot"""
|
|
|
+ from django.utils.html import format_html, mark_safe
|
|
|
+
|
|
|
+ cache_key = f'result_icons:{self.pk}:{(self.downloaded_at or self.modified_at or self.created_at or self.bookmarked_at).timestamp()}'
|
|
|
+
|
|
|
+ def calc_icons():
|
|
|
+ if hasattr(self, '_prefetched_objects_cache') and 'archiveresult_set' in self._prefetched_objects_cache:
|
|
|
+ archive_results = {r.plugin: r for r in self.archiveresult_set.all() if r.status == "succeeded" and (r.output_files or r.output_str)}
|
|
|
+ else:
|
|
|
+ # Filter for results that have either output_files or output_str
|
|
|
+ from django.db.models import Q
|
|
|
+ archive_results = {r.plugin: r for r in self.archiveresult_set.filter(
|
|
|
+ Q(status="succeeded") & (Q(output_files__isnull=False) | ~Q(output_str=''))
|
|
|
+ )}
|
|
|
+
|
|
|
+ path = self.archive_path
|
|
|
+ canon = self.canonical_outputs()
|
|
|
+ output = ""
|
|
|
+ output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a> '
|
|
|
+
|
|
|
+ # Get all plugins from hooks system (sorted by numeric prefix)
|
|
|
+ all_plugins = [get_plugin_name(e) for e in get_plugins()]
|
|
|
+
|
|
|
+ for plugin in all_plugins:
|
|
|
+ result = archive_results.get(plugin)
|
|
|
+ existing = result and result.status == 'succeeded' and (result.output_files or result.output_str)
|
|
|
+ icon = get_plugin_icon(plugin)
|
|
|
+ output += format_html(
|
|
|
+ output_template,
|
|
|
+ path,
|
|
|
+ canon.get(plugin, plugin + '/'),
|
|
|
+ str(bool(existing)),
|
|
|
+ plugin,
|
|
|
+ icon
|
|
|
+ )
|
|
|
+
|
|
|
+ return format_html('<span class="files-icons" style="font-size: 1.1em; opacity: 0.8; min-width: 240px; display: inline-block">{}</span>', mark_safe(output))
|
|
|
+
|
|
|
+ cache_result = cache.get(cache_key)
|
|
|
+ if cache_result:
|
|
|
+ return cache_result
|
|
|
+
|
|
|
+ fresh_result = calc_icons()
|
|
|
+ cache.set(cache_key, fresh_result, timeout=60 * 60 * 24)
|
|
|
+ return fresh_result
|
|
|
+
|
|
|
+ @property
|
|
|
+ def api_url(self) -> str:
|
|
|
+ return reverse_lazy('api-1:get_snapshot', args=[self.id])
|
|
|
+
|
|
|
+ def get_absolute_url(self):
|
|
|
+ return f'/{self.archive_path}'
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def domain(self) -> str:
|
|
|
+ return url_domain(self.url)
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def output_dir(self):
|
|
|
+ """The filesystem path to the snapshot's output directory."""
|
|
|
+ import os
|
|
|
+
|
|
|
+ current_path = self.get_storage_path_for_version(self.fs_version)
|
|
|
+
|
|
|
+ if current_path.exists():
|
|
|
+ return str(current_path)
|
|
|
+
|
|
|
+ # Check for backwards-compat symlink
|
|
|
+ old_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
|
|
|
+ if old_path.is_symlink():
|
|
|
+ return str(Path(os.readlink(old_path)).resolve())
|
|
|
+ elif old_path.exists():
|
|
|
+ return str(old_path)
|
|
|
+
|
|
|
+ return str(current_path)
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def archive_path(self):
|
|
|
+ return f'{CONSTANTS.ARCHIVE_DIR_NAME}/{self.timestamp}'
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def archive_size(self):
|
|
|
+ try:
|
|
|
+ return get_dir_size(self.output_dir)[0]
|
|
|
+ except Exception:
|
|
|
+ return 0
|
|
|
+
|
|
|
+ def save_tags(self, tags: Iterable[str] = ()) -> None:
|
|
|
+ tags_id = [Tag.objects.get_or_create(name=tag)[0].pk for tag in tags if tag.strip()]
|
|
|
+ self.tags.clear()
|
|
|
+ self.tags.add(*tags_id)
|
|
|
+
|
|
|
+ def pending_archiveresults(self) -> QuerySet['ArchiveResult']:
|
|
|
+ return self.archiveresult_set.exclude(status__in=ArchiveResult.FINAL_OR_ACTIVE_STATES)
|
|
|
+
|
|
|
+ def run(self) -> list['ArchiveResult']:
|
|
|
+ """
|
|
|
+ Execute snapshot by creating pending ArchiveResults for all enabled hooks.
|
|
|
+
|
|
|
+ Called by: SnapshotMachine.enter_started()
|
|
|
+
|
|
|
+ Hook Lifecycle:
|
|
|
+ 1. discover_hooks('Snapshot') → finds all plugin hooks
|
|
|
+ 2. For each hook:
|
|
|
+ - Create ArchiveResult with status=QUEUED
|
|
|
+ - Store hook_name (e.g., 'on_Snapshot__50_wget.py')
|
|
|
+ 3. ArchiveResults execute independently via ArchiveResultMachine
|
|
|
+ 4. Hook execution happens in ArchiveResult.run(), NOT here
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ list[ArchiveResult]: Newly created pending results
|
|
|
+ """
|
|
|
+ return self.create_pending_archiveresults()
|
|
|
+
|
|
|
+ def cleanup(self):
|
|
|
+ """
|
|
|
+ Clean up background ArchiveResult hooks.
|
|
|
+
|
|
|
+ Called by the state machine when entering the 'sealed' state.
|
|
|
+ Kills any background hooks and finalizes their ArchiveResults.
|
|
|
+ """
|
|
|
+ from archivebox.hooks import kill_process
|
|
|
+
|
|
|
+ # Kill any background ArchiveResult hooks
|
|
|
+ if not self.OUTPUT_DIR.exists():
|
|
|
+ return
|
|
|
+
|
|
|
+ # Find all .pid files in this snapshot's output directory
|
|
|
+ for pid_file in self.OUTPUT_DIR.glob('**/*.pid'):
|
|
|
+ kill_process(pid_file, validate=True)
|
|
|
+
|
|
|
+ # Update all STARTED ArchiveResults from filesystem
|
|
|
+ results = self.archiveresult_set.filter(status=ArchiveResult.StatusChoices.STARTED)
|
|
|
+ for ar in results:
|
|
|
+ ar.update_from_output()
|
|
|
+
|
|
|
+ def has_running_background_hooks(self) -> bool:
|
|
|
+ """
|
|
|
+ Check if any ArchiveResult background hooks are still running.
|
|
|
+
|
|
|
+ Used by state machine to determine if snapshot is finished.
|
|
|
+ """
|
|
|
+ from archivebox.hooks import process_is_alive
|
|
|
+
|
|
|
+ if not self.OUTPUT_DIR.exists():
|
|
|
+ return False
|
|
|
+
|
|
|
+ for plugin_dir in self.OUTPUT_DIR.iterdir():
|
|
|
+ if not plugin_dir.is_dir():
|
|
|
+ continue
|
|
|
+ pid_file = plugin_dir / 'hook.pid'
|
|
|
+ if process_is_alive(pid_file):
|
|
|
+ return True
|
|
|
+
|
|
|
+ return False
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def from_jsonl(record: Dict[str, Any], overrides: Dict[str, Any] = None, queue_for_extraction: bool = True):
|
|
|
+ """
|
|
|
+ Create/update Snapshot from JSONL record or dict.
|
|
|
+
|
|
|
+ Unified method that handles:
|
|
|
+ - ID-based patching: {"id": "...", "title": "new title"}
|
|
|
+ - URL-based create/update: {"url": "...", "title": "...", "tags": "..."}
|
|
|
+ - Auto-creates Crawl if not provided
|
|
|
+ - Optionally queues for extraction
|
|
|
+
|
|
|
+ Args:
|
|
|
+ record: Dict with 'url' (for create) or 'id' (for patch), plus other fields
|
|
|
+ overrides: Dict with 'crawl', 'snapshot' (parent), 'created_by_id'
|
|
|
+ queue_for_extraction: If True, sets status=QUEUED and retry_at (default: True)
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ Snapshot instance or None
|
|
|
+ """
|
|
|
+ import re
|
|
|
+ from django.utils import timezone
|
|
|
+ from archivebox.misc.util import parse_date
|
|
|
+ from archivebox.base_models.models import get_or_create_system_user_pk
|
|
|
+ from archivebox.config.common import GENERAL_CONFIG
|
|
|
+
|
|
|
+ overrides = overrides or {}
|
|
|
+
|
|
|
+ # If 'id' is provided, lookup and patch that specific snapshot
|
|
|
+ snapshot_id = record.get('id')
|
|
|
+ if snapshot_id:
|
|
|
+ try:
|
|
|
+ snapshot = Snapshot.objects.get(id=snapshot_id)
|
|
|
+
|
|
|
+ # Generically update all fields present in record
|
|
|
+ update_fields = []
|
|
|
+ for field_name, value in record.items():
|
|
|
+ # Skip internal fields
|
|
|
+ if field_name in ('id', 'type'):
|
|
|
+ continue
|
|
|
+
|
|
|
+ # Skip if field doesn't exist on model
|
|
|
+ if not hasattr(snapshot, field_name):
|
|
|
+ continue
|
|
|
+
|
|
|
+ # Special parsing for date fields
|
|
|
+ if field_name in ('bookmarked_at', 'retry_at', 'created_at', 'modified_at'):
|
|
|
+ if value and isinstance(value, str):
|
|
|
+ value = parse_date(value)
|
|
|
+
|
|
|
+ # Update field if value is provided and different
|
|
|
+ if value is not None and getattr(snapshot, field_name) != value:
|
|
|
+ setattr(snapshot, field_name, value)
|
|
|
+ update_fields.append(field_name)
|
|
|
+
|
|
|
+ if update_fields:
|
|
|
+ snapshot.save(update_fields=update_fields + ['modified_at'])
|
|
|
+
|
|
|
+ return snapshot
|
|
|
+ except Snapshot.DoesNotExist:
|
|
|
+ # ID not found, fall through to create-by-URL logic
|
|
|
+ pass
|
|
|
+
|
|
|
+ url = record.get('url')
|
|
|
+ if not url:
|
|
|
+ return None
|
|
|
+
|
|
|
+ # Determine or create crawl (every snapshot must have a crawl)
|
|
|
+ crawl = overrides.get('crawl')
|
|
|
+ parent_snapshot = overrides.get('snapshot') # Parent snapshot
|
|
|
+ created_by_id = overrides.get('created_by_id') or (parent_snapshot.crawl.created_by_id if parent_snapshot else None) or get_or_create_system_user_pk()
|
|
|
+
|
|
|
+ # If no crawl provided, inherit from parent or auto-create one
|
|
|
+ if not crawl:
|
|
|
+ if parent_snapshot:
|
|
|
+ # Inherit crawl from parent snapshot
|
|
|
+ crawl = parent_snapshot.crawl
|
|
|
+ else:
|
|
|
+ # Auto-create a single-URL crawl
|
|
|
+ from archivebox.crawls.models import Crawl
|
|
|
+ from archivebox.config import CONSTANTS
|
|
|
+
|
|
|
+ timestamp_str = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
|
|
|
+ sources_file = CONSTANTS.SOURCES_DIR / f'{timestamp_str}__auto_crawl.txt'
|
|
|
+ sources_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
+ sources_file.write_text(url)
|
|
|
+
|
|
|
+ crawl = Crawl.objects.create(
|
|
|
+ urls=url,
|
|
|
+ max_depth=0,
|
|
|
+ label=f'auto-created for {url[:50]}',
|
|
|
+ created_by_id=created_by_id,
|
|
|
+ )
|
|
|
+
|
|
|
+ # Parse tags
|
|
|
+ tags_str = record.get('tags', '')
|
|
|
+ tag_list = []
|
|
|
+ if tags_str:
|
|
|
+ tag_list = list(dict.fromkeys(
|
|
|
+ tag.strip() for tag in re.split(GENERAL_CONFIG.TAG_SEPARATOR_PATTERN, tags_str)
|
|
|
+ if tag.strip()
|
|
|
+ ))
|
|
|
+
|
|
|
+ # Get most recent snapshot with this URL (URLs can exist in multiple crawls)
|
|
|
+ snapshot = Snapshot.objects.filter(url=url).order_by('-created_at').first()
|
|
|
+
|
|
|
+ title = record.get('title')
|
|
|
+ timestamp = record.get('timestamp')
|
|
|
+
|
|
|
+ if snapshot:
|
|
|
+ # Update existing snapshot
|
|
|
+ if title and (not snapshot.title or len(title) > len(snapshot.title or '')):
|
|
|
+ snapshot.title = title
|
|
|
+ snapshot.save(update_fields=['title', 'modified_at'])
|
|
|
+ else:
|
|
|
+ # Create new snapshot
|
|
|
+ if timestamp:
|
|
|
+ while Snapshot.objects.filter(timestamp=timestamp).exists():
|
|
|
+ timestamp = str(float(timestamp) + 1.0)
|
|
|
+
|
|
|
+ snapshot = Snapshot.objects.create(
|
|
|
+ url=url,
|
|
|
+ timestamp=timestamp,
|
|
|
+ title=title,
|
|
|
+ crawl=crawl,
|
|
|
+ )
|
|
|
+
|
|
|
+ # Update tags
|
|
|
+ if tag_list:
|
|
|
+ existing_tags = set(snapshot.tags.values_list('name', flat=True))
|
|
|
+ new_tags = set(tag_list) | existing_tags
|
|
|
+ snapshot.save_tags(new_tags)
|
|
|
+
|
|
|
+ # Queue for extraction and update additional fields
|
|
|
+ update_fields = []
|
|
|
+
|
|
|
+ if queue_for_extraction:
|
|
|
+ snapshot.status = Snapshot.StatusChoices.QUEUED
|
|
|
+ snapshot.retry_at = timezone.now()
|
|
|
+ update_fields.extend(['status', 'retry_at'])
|
|
|
+
|
|
|
+ # Update additional fields if provided
|
|
|
+ for field_name in ('depth', 'parent_snapshot_id', 'crawl_id', 'bookmarked_at'):
|
|
|
+ value = record.get(field_name)
|
|
|
+ if value is not None and getattr(snapshot, field_name) != value:
|
|
|
+ setattr(snapshot, field_name, value)
|
|
|
+ update_fields.append(field_name)
|
|
|
+
|
|
|
+ if update_fields:
|
|
|
+ snapshot.save(update_fields=update_fields + ['modified_at'])
|
|
|
+
|
|
|
+ return snapshot
|
|
|
+
|
|
|
+ def create_pending_archiveresults(self) -> list['ArchiveResult']:
|
|
|
+ """
|
|
|
+ Create ArchiveResult records for all enabled hooks.
|
|
|
+
|
|
|
+ Uses the hooks system to discover available hooks from:
|
|
|
+ - archivebox/plugins/*/on_Snapshot__*.{py,sh,js}
|
|
|
+ - data/plugins/*/on_Snapshot__*.{py,sh,js}
|
|
|
+
|
|
|
+ Creates one ArchiveResult per hook (not per plugin), with hook_name set.
|
|
|
+ This enables step-based execution where all hooks in a step can run in parallel.
|
|
|
+ """
|
|
|
+ from archivebox.hooks import discover_hooks
|
|
|
+
|
|
|
+ hooks = discover_hooks('Snapshot')
|
|
|
+ archiveresults = []
|
|
|
+
|
|
|
+ for hook_path in hooks:
|
|
|
+ hook_name = hook_path.name # e.g., 'on_Snapshot__50_wget.py'
|
|
|
+ plugin = hook_path.parent.name # e.g., 'wget'
|
|
|
+
|
|
|
+ # Check if AR already exists for this specific hook
|
|
|
+ if ArchiveResult.objects.filter(snapshot=self, hook_name=hook_name).exists():
|
|
|
+ continue
|
|
|
+
|
|
|
+ archiveresult, created = ArchiveResult.objects.get_or_create(
|
|
|
+ snapshot=self,
|
|
|
+ hook_name=hook_name,
|
|
|
+ defaults={
|
|
|
+ 'plugin': plugin,
|
|
|
+ 'status': ArchiveResult.INITIAL_STATE,
|
|
|
+ 'retry_at': timezone.now(),
|
|
|
+ 'created_by_id': self.crawl.created_by_id,
|
|
|
+ },
|
|
|
+ )
|
|
|
+ if archiveresult.status == ArchiveResult.INITIAL_STATE:
|
|
|
+ archiveresults.append(archiveresult)
|
|
|
+
|
|
|
+ return archiveresults
|
|
|
+
|
|
|
+ def advance_step_if_ready(self) -> bool:
|
|
|
+ """
|
|
|
+ Advance current_step if all foreground hooks in current step are finished.
|
|
|
+
|
|
|
+ Called by the state machine to check if step can advance.
|
|
|
+ Background hooks (.bg) don't block step advancement.
|
|
|
+
|
|
|
+ Step advancement rules:
|
|
|
+ - All foreground ARs in current step must be finished (SUCCEEDED/FAILED/SKIPPED)
|
|
|
+ - Background ARs (hook_name contains '.bg.') are ignored for advancement
|
|
|
+ - When ready, increments current_step by 1 (up to 9)
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ True if step was advanced, False if not ready or already at step 9.
|
|
|
+ """
|
|
|
+ from archivebox.hooks import extract_step, is_background_hook
|
|
|
+
|
|
|
+ if self.current_step >= 9:
|
|
|
+ return False # Already at final step
|
|
|
+
|
|
|
+ # Get all ARs for current step that are foreground
|
|
|
+ current_step_ars = self.archiveresult_set.filter(
|
|
|
+ hook_name__isnull=False
|
|
|
+ ).exclude(hook_name='')
|
|
|
+
|
|
|
+ # Check each AR in current step
|
|
|
+ for ar in current_step_ars:
|
|
|
+ ar_step = extract_step(ar.hook_name)
|
|
|
+ if ar_step != self.current_step:
|
|
|
+ continue # Not in current step
|
|
|
+
|
|
|
+ if is_background_hook(ar.hook_name):
|
|
|
+ continue # Background hooks don't block
|
|
|
+
|
|
|
+ # Foreground hook in current step - check if finished
|
|
|
+ if ar.status not in ArchiveResult.FINAL_OR_ACTIVE_STATES:
|
|
|
+ # Still pending/queued - can't advance
|
|
|
+ return False
|
|
|
+
|
|
|
+ if ar.status == ArchiveResult.StatusChoices.STARTED:
|
|
|
+ # Still running - can't advance
|
|
|
+ return False
|
|
|
+
|
|
|
+ # All foreground hooks in current step are finished - advance!
|
|
|
+ self.current_step += 1
|
|
|
+ self.save(update_fields=['current_step', 'modified_at'])
|
|
|
+ return True
|
|
|
+
|
|
|
+ def is_finished_processing(self) -> bool:
|
|
|
+ """
|
|
|
+ Check if this snapshot has finished processing.
|
|
|
+
|
|
|
+ Used by SnapshotMachine.is_finished() to determine if snapshot is complete.
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ True if all archiveresults are finished (or no work to do), False otherwise.
|
|
|
+ """
|
|
|
+ # if no archiveresults exist yet, it's not finished
|
|
|
+ if not self.archiveresult_set.exists():
|
|
|
+ return False
|
|
|
+
|
|
|
+ # Try to advance step if ready (handles step-based hook execution)
|
|
|
+ # This will increment current_step when all foreground hooks in current step are done
|
|
|
+ while self.advance_step_if_ready():
|
|
|
+ pass # Keep advancing until we can't anymore
|
|
|
+
|
|
|
+ # if archiveresults exist but are still pending, it's not finished
|
|
|
+ if self.pending_archiveresults().exists():
|
|
|
+ return False
|
|
|
+
|
|
|
+ # Don't wait for background hooks - they'll be cleaned up on entering sealed state
|
|
|
+ # Background hooks in STARTED state are excluded by pending_archiveresults()
|
|
|
+ # (STARTED is in FINAL_OR_ACTIVE_STATES) so once all results are FINAL or ACTIVE,
|
|
|
+ # we can transition to sealed and cleanup() will kill the background hooks
|
|
|
+
|
|
|
+ # otherwise archiveresults exist and are all finished, so it's finished
|
|
|
+ return True
|
|
|
+
|
|
|
+ def retry_failed_archiveresults(self, retry_at: Optional['timezone.datetime'] = None) -> int:
|
|
|
+ """
|
|
|
+ Reset failed/skipped ArchiveResults to queued for retry.
|
|
|
+
|
|
|
+ This enables seamless retry of the entire extraction pipeline:
|
|
|
+ - Resets FAILED and SKIPPED results to QUEUED
|
|
|
+ - Sets retry_at so workers pick them up
|
|
|
+ - Plugins run in order (numeric prefix)
|
|
|
+ - Each plugin checks its dependencies at runtime
|
|
|
+
|
|
|
+ Dependency handling (e.g., chrome_session → screenshot):
|
|
|
+ - Plugins check if required outputs exist before running
|
|
|
+ - If dependency output missing → plugin returns 'skipped'
|
|
|
+ - On retry, if dependency now succeeds → dependent can run
|
|
|
+
|
|
|
+ Returns count of ArchiveResults reset.
|
|
|
+ """
|
|
|
+ retry_at = retry_at or timezone.now()
|
|
|
+
|
|
|
+ count = self.archiveresult_set.filter(
|
|
|
+ status__in=[
|
|
|
+ ArchiveResult.StatusChoices.FAILED,
|
|
|
+ ArchiveResult.StatusChoices.SKIPPED,
|
|
|
+ ]
|
|
|
+ ).update(
|
|
|
+ status=ArchiveResult.StatusChoices.QUEUED,
|
|
|
+ retry_at=retry_at,
|
|
|
+ output=None,
|
|
|
+ start_ts=None,
|
|
|
+ end_ts=None,
|
|
|
+ )
|
|
|
+
|
|
|
+ # Also reset the snapshot and current_step so it gets re-checked from the beginning
|
|
|
+ if count > 0:
|
|
|
+ self.status = self.StatusChoices.STARTED
|
|
|
+ self.retry_at = retry_at
|
|
|
+ self.current_step = 0 # Reset to step 0 for retry
|
|
|
+ self.save(update_fields=['status', 'retry_at', 'current_step', 'modified_at'])
|
|
|
+
|
|
|
+ return count
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # URL Helper Properties (migrated from Link schema)
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def url_hash(self) -> str:
|
|
|
+ from hashlib import sha256
|
|
|
+ return sha256(self.url.encode()).hexdigest()[:8]
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def scheme(self) -> str:
|
|
|
+ return self.url.split('://')[0]
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def path(self) -> str:
|
|
|
+ parts = self.url.split('://', 1)
|
|
|
+ return '/' + parts[1].split('/', 1)[1] if len(parts) > 1 and '/' in parts[1] else '/'
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def basename(self) -> str:
|
|
|
+ return self.path.split('/')[-1]
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def extension(self) -> str:
|
|
|
+ basename = self.basename
|
|
|
+ return basename.split('.')[-1] if '.' in basename else ''
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def base_url(self) -> str:
|
|
|
+ return f'{self.scheme}://{self.domain}'
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def is_static(self) -> bool:
|
|
|
+ static_extensions = {'.pdf', '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.mp4', '.mp3', '.wav', '.webm'}
|
|
|
+ return any(self.url.lower().endswith(ext) for ext in static_extensions)
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def is_archived(self) -> bool:
|
|
|
+ output_paths = (
|
|
|
+ self.domain,
|
|
|
+ 'output.html',
|
|
|
+ 'output.pdf',
|
|
|
+ 'screenshot.png',
|
|
|
+ 'singlefile.html',
|
|
|
+ 'readability/content.html',
|
|
|
+ 'mercury/content.html',
|
|
|
+ 'htmltotext.txt',
|
|
|
+ 'media',
|
|
|
+ 'git',
|
|
|
+ )
|
|
|
+ return any((Path(self.output_dir) / path).exists() for path in output_paths)
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Date/Time Properties (migrated from Link schema)
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def bookmarked_date(self) -> Optional[str]:
|
|
|
+ max_ts = (timezone.now() + timedelta(days=30)).timestamp()
|
|
|
+ if self.timestamp and self.timestamp.replace('.', '').isdigit():
|
|
|
+ if 0 < float(self.timestamp) < max_ts:
|
|
|
+ return self._ts_to_date_str(datetime.fromtimestamp(float(self.timestamp)))
|
|
|
+ return str(self.timestamp)
|
|
|
+ return None
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def downloaded_datestr(self) -> Optional[str]:
|
|
|
+ return self._ts_to_date_str(self.downloaded_at) if self.downloaded_at else None
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def archive_dates(self) -> List[datetime]:
|
|
|
+ return [
|
|
|
+ result.start_ts
|
|
|
+ for result in self.archiveresult_set.all()
|
|
|
+ if result.start_ts
|
|
|
+ ]
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def oldest_archive_date(self) -> Optional[datetime]:
|
|
|
+ dates = self.archive_dates
|
|
|
+ return min(dates) if dates else None
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def newest_archive_date(self) -> Optional[datetime]:
|
|
|
+ dates = self.archive_dates
|
|
|
+ return max(dates) if dates else None
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def num_outputs(self) -> int:
|
|
|
+ return self.archiveresult_set.filter(status='succeeded').count()
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def num_failures(self) -> int:
|
|
|
+ return self.archiveresult_set.filter(status='failed').count()
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Output Path Methods (migrated from Link schema)
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ def canonical_outputs(self) -> Dict[str, Optional[str]]:
|
|
|
+ """
|
|
|
+ Intelligently discover the best output file for each plugin.
|
|
|
+ Uses actual ArchiveResult data and filesystem scanning with smart heuristics.
|
|
|
+ """
|
|
|
+ FAVICON_PROVIDER = 'https://www.google.com/s2/favicons?domain={}'
|
|
|
+
|
|
|
+ # Mimetypes that can be embedded/previewed in an iframe
|
|
|
+ IFRAME_EMBEDDABLE_EXTENSIONS = {
|
|
|
+ 'html', 'htm', 'pdf', 'txt', 'md', 'json', 'jsonl',
|
|
|
+ 'png', 'jpg', 'jpeg', 'gif', 'webp', 'svg', 'ico',
|
|
|
+ 'mp4', 'webm', 'mp3', 'opus', 'ogg', 'wav',
|
|
|
+ }
|
|
|
+
|
|
|
+ MIN_DISPLAY_SIZE = 15_000 # 15KB - filter out tiny files
|
|
|
+ MAX_SCAN_FILES = 50 # Don't scan massive directories
|
|
|
+
|
|
|
+ def find_best_output_in_dir(dir_path: Path, plugin_name: str) -> Optional[str]:
|
|
|
+ """Find the best representative file in a plugin's output directory"""
|
|
|
+ if not dir_path.exists() or not dir_path.is_dir():
|
|
|
+ return None
|
|
|
+
|
|
|
+ candidates = []
|
|
|
+ file_count = 0
|
|
|
+
|
|
|
+ # Special handling for media plugin - look for thumbnails
|
|
|
+ is_media_dir = plugin_name == 'media'
|
|
|
+
|
|
|
+ # Scan for suitable files
|
|
|
+ for file_path in dir_path.rglob('*'):
|
|
|
+ file_count += 1
|
|
|
+ if file_count > MAX_SCAN_FILES:
|
|
|
+ break
|
|
|
+
|
|
|
+ if file_path.is_dir() or file_path.name.startswith('.'):
|
|
|
+ continue
|
|
|
+
|
|
|
+ ext = file_path.suffix.lstrip('.').lower()
|
|
|
+ if ext not in IFRAME_EMBEDDABLE_EXTENSIONS:
|
|
|
+ continue
|
|
|
+
|
|
|
+ try:
|
|
|
+ size = file_path.stat().st_size
|
|
|
+ except OSError:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # For media dir, allow smaller image files (thumbnails are often < 15KB)
|
|
|
+ min_size = 5_000 if (is_media_dir and ext in ('png', 'jpg', 'jpeg', 'webp', 'gif')) else MIN_DISPLAY_SIZE
|
|
|
+ if size < min_size:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # Prefer main files: index.html, output.*, content.*, etc.
|
|
|
+ priority = 0
|
|
|
+ name_lower = file_path.name.lower()
|
|
|
+
|
|
|
+ if is_media_dir:
|
|
|
+ # Special prioritization for media directories
|
|
|
+ if any(keyword in name_lower for keyword in ('thumb', 'thumbnail', 'cover', 'poster')):
|
|
|
+ priority = 200 # Highest priority for thumbnails
|
|
|
+ elif ext in ('png', 'jpg', 'jpeg', 'webp', 'gif'):
|
|
|
+ priority = 150 # High priority for any image
|
|
|
+ elif ext in ('mp4', 'webm', 'mp3', 'opus', 'ogg'):
|
|
|
+ priority = 100 # Lower priority for actual media files
|
|
|
+ else:
|
|
|
+ priority = 50
|
|
|
+ elif 'index' in name_lower:
|
|
|
+ priority = 100
|
|
|
+ elif name_lower.startswith(('output', 'content', plugin_name)):
|
|
|
+ priority = 50
|
|
|
+ elif ext in ('html', 'htm', 'pdf'):
|
|
|
+ priority = 30
|
|
|
+ elif ext in ('png', 'jpg', 'jpeg', 'webp'):
|
|
|
+ priority = 20
|
|
|
+ else:
|
|
|
+ priority = 10
|
|
|
+
|
|
|
+ candidates.append((priority, size, file_path))
|
|
|
+
|
|
|
+ if not candidates:
|
|
|
+ return None
|
|
|
+
|
|
|
+ # Sort by priority (desc), then size (desc)
|
|
|
+ candidates.sort(key=lambda x: (x[0], x[1]), reverse=True)
|
|
|
+ best_file = candidates[0][2]
|
|
|
+ return str(best_file.relative_to(Path(self.output_dir)))
|
|
|
+
|
|
|
+ canonical = {
|
|
|
+ 'index_path': 'index.html',
|
|
|
+ 'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
|
|
|
+ 'archive_org_path': f'https://web.archive.org/web/{self.base_url}',
|
|
|
+ }
|
|
|
+
|
|
|
+ # Scan each ArchiveResult's output directory for the best file
|
|
|
+ snap_dir = Path(self.output_dir)
|
|
|
+ for result in self.archiveresult_set.filter(status='succeeded'):
|
|
|
+ if not result.output_files and not result.output_str:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # Try to find the best output file for this plugin
|
|
|
+ plugin_dir = snap_dir / result.plugin
|
|
|
+ best_output = None
|
|
|
+
|
|
|
+ # Check output_files first (new field)
|
|
|
+ if result.output_files:
|
|
|
+ first_file = next(iter(result.output_files.keys()), None)
|
|
|
+ if first_file and (plugin_dir / first_file).exists():
|
|
|
+ best_output = f'{result.plugin}/{first_file}'
|
|
|
+
|
|
|
+ # Fallback to output_str if it looks like a path
|
|
|
+ if not best_output and result.output_str and (snap_dir / result.output_str).exists():
|
|
|
+ best_output = result.output_str
|
|
|
+
|
|
|
+ if not best_output and plugin_dir.exists():
|
|
|
+ # Intelligently find the best file in the plugin's directory
|
|
|
+ best_output = find_best_output_in_dir(plugin_dir, result.plugin)
|
|
|
+
|
|
|
+ if best_output:
|
|
|
+ canonical[f'{result.plugin}_path'] = best_output
|
|
|
+
|
|
|
+ # Also scan top-level for legacy outputs (backwards compatibility)
|
|
|
+ for file_path in snap_dir.glob('*'):
|
|
|
+ if file_path.is_dir() or file_path.name in ('index.html', 'index.json'):
|
|
|
+ continue
|
|
|
+
|
|
|
+ ext = file_path.suffix.lstrip('.').lower()
|
|
|
+ if ext not in IFRAME_EMBEDDABLE_EXTENSIONS:
|
|
|
+ continue
|
|
|
+
|
|
|
+ try:
|
|
|
+ size = file_path.stat().st_size
|
|
|
+ if size >= MIN_DISPLAY_SIZE:
|
|
|
+ # Add as generic output with stem as key
|
|
|
+ key = f'{file_path.stem}_path'
|
|
|
+ if key not in canonical:
|
|
|
+ canonical[key] = file_path.name
|
|
|
+ except OSError:
|
|
|
+ continue
|
|
|
+
|
|
|
+ if self.is_static:
|
|
|
+ static_path = f'warc/{self.timestamp}'
|
|
|
+ canonical.update({
|
|
|
+ 'title': self.basename,
|
|
|
+ 'wget_path': static_path,
|
|
|
+ })
|
|
|
+
|
|
|
+ return canonical
|
|
|
+
|
|
|
+ def latest_outputs(self, status: Optional[str] = None) -> Dict[str, Any]:
|
|
|
+ """Get the latest output that each plugin produced"""
|
|
|
+ from archivebox.hooks import get_plugins
|
|
|
+ from django.db.models import Q
|
|
|
+
|
|
|
+ latest: Dict[str, Any] = {}
|
|
|
+ for plugin in get_plugins():
|
|
|
+ results = self.archiveresult_set.filter(plugin=plugin)
|
|
|
+ if status is not None:
|
|
|
+ results = results.filter(status=status)
|
|
|
+ # Filter for results with output_files or output_str
|
|
|
+ results = results.filter(Q(output_files__isnull=False) | ~Q(output_str='')).order_by('-start_ts')
|
|
|
+ result = results.first()
|
|
|
+ # Return embed_path() for backwards compatibility
|
|
|
+ latest[plugin] = result.embed_path() if result else None
|
|
|
+ return latest
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Serialization Methods
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ def to_dict(self, extended: bool = False) -> Dict[str, Any]:
|
|
|
+ """Convert Snapshot to a dictionary (replacement for Link._asdict())"""
|
|
|
+ from archivebox.misc.util import ts_to_date_str
|
|
|
+
|
|
|
+ result = {
|
|
|
+ 'TYPE': 'core.models.Snapshot',
|
|
|
+ 'id': str(self.id),
|
|
|
+ 'url': self.url,
|
|
|
+ 'timestamp': self.timestamp,
|
|
|
+ 'title': self.title,
|
|
|
+ 'tags': self.tags_str(),
|
|
|
+ 'downloaded_at': self.downloaded_at.isoformat() if self.downloaded_at else None,
|
|
|
+ 'bookmarked_at': self.bookmarked_at.isoformat() if self.bookmarked_at else None,
|
|
|
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
|
|
|
+ # Computed properties
|
|
|
+ 'domain': self.domain,
|
|
|
+ 'scheme': self.scheme,
|
|
|
+ 'base_url': self.base_url,
|
|
|
+ 'path': self.path,
|
|
|
+ 'basename': self.basename,
|
|
|
+ 'extension': self.extension,
|
|
|
+ 'is_static': self.is_static,
|
|
|
+ 'is_archived': self.is_archived,
|
|
|
+ 'archive_path': self.archive_path,
|
|
|
+ 'output_dir': self.output_dir,
|
|
|
+ 'link_dir': self.output_dir, # backwards compatibility alias
|
|
|
+ 'archive_size': self.archive_size,
|
|
|
+ 'bookmarked_date': self.bookmarked_date,
|
|
|
+ 'downloaded_datestr': self.downloaded_datestr,
|
|
|
+ 'num_outputs': self.num_outputs,
|
|
|
+ 'num_failures': self.num_failures,
|
|
|
+ }
|
|
|
+ if extended:
|
|
|
+ result['canonical'] = self.canonical_outputs()
|
|
|
+ return result
|
|
|
+
|
|
|
+ def to_json(self, indent: int = 4) -> str:
|
|
|
+ """Convert to JSON string"""
|
|
|
+ return to_json(self.to_dict(extended=True), indent=indent)
|
|
|
+
|
|
|
+ def to_csv(self, cols: Optional[List[str]] = None, separator: str = ',', ljust: int = 0) -> str:
|
|
|
+ """Convert to CSV string"""
|
|
|
+ data = self.to_dict()
|
|
|
+ cols = cols or ['timestamp', 'is_archived', 'url']
|
|
|
+ return separator.join(to_json(data.get(col, ''), indent=None).ljust(ljust) for col in cols)
|
|
|
+
|
|
|
+ def write_json_details(self, out_dir: Optional[str] = None) -> None:
|
|
|
+ """Write JSON index file for this snapshot to its output directory"""
|
|
|
+ out_dir = out_dir or self.output_dir
|
|
|
+ path = Path(out_dir) / CONSTANTS.JSON_INDEX_FILENAME
|
|
|
+ atomic_write(str(path), self.to_dict(extended=True))
|
|
|
+
|
|
|
+ def write_html_details(self, out_dir: Optional[str] = None) -> None:
|
|
|
+ """Write HTML detail page for this snapshot to its output directory"""
|
|
|
+ from django.template.loader import render_to_string
|
|
|
+ from archivebox.config.common import SERVER_CONFIG
|
|
|
+ from archivebox.config.configset import get_config
|
|
|
+ from archivebox.misc.logging_util import printable_filesize
|
|
|
+
|
|
|
+ out_dir = out_dir or self.output_dir
|
|
|
+ config = get_config()
|
|
|
+ SAVE_ARCHIVE_DOT_ORG = config.get('SAVE_ARCHIVE_DOT_ORG', True)
|
|
|
+ TITLE_LOADING_MSG = 'Not yet archived...'
|
|
|
+
|
|
|
+ canonical = self.canonical_outputs()
|
|
|
+ context = {
|
|
|
+ **self.to_dict(extended=True),
|
|
|
+ **{f'{k}_path': v for k, v in canonical.items()},
|
|
|
+ 'canonical': {f'{k}_path': v for k, v in canonical.items()},
|
|
|
+ 'title': htmlencode(self.title or (self.base_url if self.is_archived else TITLE_LOADING_MSG)),
|
|
|
+ 'url_str': htmlencode(urldecode(self.base_url)),
|
|
|
+ 'archive_url': urlencode(f'warc/{self.timestamp}' or (self.domain if self.is_archived else '')) or 'about:blank',
|
|
|
+ 'extension': self.extension or 'html',
|
|
|
+ 'tags': self.tags_str() or 'untagged',
|
|
|
+ 'size': printable_filesize(self.archive_size) if self.archive_size else 'pending',
|
|
|
+ 'status': 'archived' if self.is_archived else 'not yet archived',
|
|
|
+ 'status_color': 'success' if self.is_archived else 'danger',
|
|
|
+ 'oldest_archive_date': ts_to_date_str(self.oldest_archive_date),
|
|
|
+ 'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
|
|
|
+ 'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
|
|
|
+ }
|
|
|
+ rendered_html = render_to_string('snapshot.html', context)
|
|
|
+ atomic_write(str(Path(out_dir) / CONSTANTS.HTML_INDEX_FILENAME), rendered_html)
|
|
|
+
|
|
|
+ # =========================================================================
|
|
|
+ # Helper Methods
|
|
|
+ # =========================================================================
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _ts_to_date_str(dt: Optional[datetime]) -> Optional[str]:
|
|
|
+ return dt.strftime('%Y-%m-%d %H:%M:%S') if dt else None
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Snapshot State Machine
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+class SnapshotMachine(BaseStateMachine, strict_states=True):
|
|
|
+ """
|
|
|
+ State machine for managing Snapshot lifecycle.
|
|
|
+
|
|
|
+ Hook Lifecycle:
|
|
|
+ ┌─────────────────────────────────────────────────────────────┐
|
|
|
+ │ QUEUED State │
|
|
|
+ │ • Waiting for snapshot to be ready │
|
|
|
+ └─────────────────────────────────────────────────────────────┘
|
|
|
+ ↓ tick() when can_start()
|
|
|
+ ┌─────────────────────────────────────────────────────────────┐
|
|
|
+ │ STARTED State → enter_started() │
|
|
|
+ │ 1. snapshot.run() │
|
|
|
+ │ • discover_hooks('Snapshot') → finds all plugin hooks │
|
|
|
+ │ • create_pending_archiveresults() → creates ONE │
|
|
|
+ │ ArchiveResult per hook (NO execution yet) │
|
|
|
+ │ 2. ArchiveResults process independently with their own │
|
|
|
+ │ state machines (see ArchiveResultMachine) │
|
|
|
+ │ 3. Advance through steps 0-9 as foreground hooks complete │
|
|
|
+ └─────────────────────────────────────────────────────────────┘
|
|
|
+ ↓ tick() when is_finished()
|
|
|
+ ┌─────────────────────────────────────────────────────────────┐
|
|
|
+ │ SEALED State → enter_sealed() │
|
|
|
+ │ • cleanup() → kills any background hooks still running │
|
|
|
+ │ • Set retry_at=None (no more processing) │
|
|
|
+ └─────────────────────────────────────────────────────────────┘
|
|
|
+
|
|
|
+ https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
|
|
|
+ """
|
|
|
+
|
|
|
+ model_attr_name = 'snapshot'
|
|
|
+
|
|
|
+ # States
|
|
|
+ queued = State(value=Snapshot.StatusChoices.QUEUED, initial=True)
|
|
|
+ started = State(value=Snapshot.StatusChoices.STARTED)
|
|
|
+ sealed = State(value=Snapshot.StatusChoices.SEALED, final=True)
|
|
|
+
|
|
|
+ # Tick Event
|
|
|
+ tick = (
|
|
|
+ queued.to.itself(unless='can_start') |
|
|
|
+ queued.to(started, cond='can_start') |
|
|
|
+ started.to.itself(unless='is_finished') |
|
|
|
+ started.to(sealed, cond='is_finished')
|
|
|
+ )
|
|
|
+
|
|
|
+ def can_start(self) -> bool:
|
|
|
+ can_start = bool(self.snapshot.url)
|
|
|
+ # Suppressed: queue waiting logs
|
|
|
+ return can_start
|
|
|
+
|
|
|
+ def is_finished(self) -> bool:
|
|
|
+ """Check if snapshot processing is complete - delegates to model method."""
|
|
|
+ return self.snapshot.is_finished_processing()
|
|
|
+
|
|
|
+ @queued.enter
|
|
|
+ def enter_queued(self):
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ self.snapshot.update_and_requeue(
|
|
|
+ retry_at=timezone.now(),
|
|
|
+ status=Snapshot.StatusChoices.QUEUED,
|
|
|
+ )
|
|
|
+
|
|
|
+ @started.enter
|
|
|
+ def enter_started(self):
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ # lock the snapshot while we create the pending archiveresults
|
|
|
+ self.snapshot.update_and_requeue(
|
|
|
+ retry_at=timezone.now() + timedelta(seconds=30), # if failed, wait 30s before retrying
|
|
|
+ )
|
|
|
+
|
|
|
+ # Run the snapshot - creates pending archiveresults for all enabled plugins
|
|
|
+ self.snapshot.run()
|
|
|
+
|
|
|
+ # unlock the snapshot after we're done + set status = started
|
|
|
+ self.snapshot.update_and_requeue(
|
|
|
+ retry_at=timezone.now() + timedelta(seconds=5), # check again in 5s
|
|
|
+ status=Snapshot.StatusChoices.STARTED,
|
|
|
+ )
|
|
|
+
|
|
|
+ @sealed.enter
|
|
|
+ def enter_sealed(self):
|
|
|
+ # Clean up background hooks
|
|
|
+ self.snapshot.cleanup()
|
|
|
+
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ self.snapshot.update_and_requeue(
|
|
|
+ retry_at=None,
|
|
|
+ status=Snapshot.StatusChoices.SEALED,
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+class ArchiveResultManager(models.Manager):
|
|
|
+ def indexable(self, sorted: bool = True):
|
|
|
+ INDEXABLE_METHODS = [r[0] for r in EXTRACTOR_INDEXING_PRECEDENCE]
|
|
|
+ qs = self.get_queryset().filter(plugin__in=INDEXABLE_METHODS, status='succeeded')
|
|
|
+ if sorted:
|
|
|
+ precedence = [When(plugin=method, then=Value(p)) for method, p in EXTRACTOR_INDEXING_PRECEDENCE]
|
|
|
+ qs = qs.annotate(indexing_precedence=Case(*precedence, default=Value(1000), output_field=IntegerField())).order_by('indexing_precedence')
|
|
|
+ return qs
|
|
|
+
|
|
|
+
|
|
|
+class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithStateMachine):
|
|
|
+ class StatusChoices(models.TextChoices):
|
|
|
+ QUEUED = 'queued', 'Queued'
|
|
|
+ STARTED = 'started', 'Started'
|
|
|
+ BACKOFF = 'backoff', 'Waiting to retry'
|
|
|
+ SUCCEEDED = 'succeeded', 'Succeeded'
|
|
|
+ FAILED = 'failed', 'Failed'
|
|
|
+ SKIPPED = 'skipped', 'Skipped'
|
|
|
+
|
|
|
+ @classmethod
|
|
|
+ def get_plugin_choices(cls):
|
|
|
+ """Get plugin choices from discovered hooks (for forms/admin)."""
|
|
|
+ plugins = [get_plugin_name(e) for e in get_plugins()]
|
|
|
+ return tuple((e, e) for e in plugins)
|
|
|
+
|
|
|
+ # Keep AutoField for backward compatibility with 0.7.x databases
|
|
|
+ # UUID field is added separately by migration for new records
|
|
|
+ id = models.AutoField(primary_key=True, editable=False)
|
|
|
+ # Note: unique constraint is added by migration 0027 - don't set unique=True here
|
|
|
+ # or SQLite table recreation in earlier migrations will fail
|
|
|
+ uuid = models.UUIDField(default=uuid7, null=True, blank=True, db_index=True)
|
|
|
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False, related_name='archiveresult_set', db_index=True)
|
|
|
+ created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
|
|
+ modified_at = models.DateTimeField(auto_now=True)
|
|
|
+
|
|
|
+ snapshot: Snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) # type: ignore
|
|
|
+ # No choices= constraint - plugin names come from plugin system and can be any string
|
|
|
+ plugin = models.CharField(max_length=32, blank=False, null=False, db_index=True)
|
|
|
+ hook_name = models.CharField(max_length=255, blank=True, default='', db_index=True, help_text='Full filename of the hook that executed (e.g., on_Snapshot__50_wget.py)')
|
|
|
+ pwd = models.CharField(max_length=256, default=None, null=True, blank=True)
|
|
|
+ cmd = models.JSONField(default=None, null=True, blank=True)
|
|
|
+ cmd_version = models.CharField(max_length=128, default=None, null=True, blank=True)
|
|
|
+
|
|
|
+ # New output fields (replacing old 'output' field)
|
|
|
+ output_str = models.TextField(blank=True, default='', help_text='Human-readable output summary')
|
|
|
+ output_json = models.JSONField(null=True, blank=True, default=None, help_text='Structured metadata (headers, redirects, etc.)')
|
|
|
+ output_files = models.JSONField(default=dict, help_text='Dict of {relative_path: {metadata}}')
|
|
|
+ output_size = models.BigIntegerField(default=0, help_text='Total bytes of all output files')
|
|
|
+ output_mimetypes = models.CharField(max_length=512, blank=True, default='', help_text='CSV of mimetypes sorted by size')
|
|
|
+
|
|
|
+ # Binary FK (optional - set when hook reports cmd)
|
|
|
+ binary = models.ForeignKey(
|
|
|
+ 'machine.Binary',
|
|
|
+ on_delete=models.SET_NULL,
|
|
|
+ null=True, blank=True,
|
|
|
+ related_name='archiveresults',
|
|
|
+ help_text='Primary binary used by this hook'
|
|
|
+ )
|
|
|
+
|
|
|
+ start_ts = models.DateTimeField(default=None, null=True, blank=True)
|
|
|
+ end_ts = models.DateTimeField(default=None, null=True, blank=True)
|
|
|
+
|
|
|
+ status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED)
|
|
|
+ retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
|
|
|
+ notes = models.TextField(blank=True, null=False, default='')
|
|
|
+ output_dir = models.CharField(max_length=256, default=None, null=True, blank=True)
|
|
|
+ iface = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True)
|
|
|
+
|
|
|
+ state_machine_name = 'core.models.ArchiveResultMachine'
|
|
|
+ retry_at_field_name = 'retry_at'
|
|
|
+ state_field_name = 'status'
|
|
|
+ active_state = StatusChoices.STARTED
|
|
|
+
|
|
|
+ objects = ArchiveResultManager()
|
|
|
+
|
|
|
+ class Meta(TypedModelMeta):
|
|
|
+ verbose_name = 'Archive Result'
|
|
|
+ verbose_name_plural = 'Archive Results Log'
|
|
|
+
|
|
|
+ def __str__(self):
|
|
|
+ return f'[{self.id}] {self.snapshot.url[:64]} -> {self.plugin}'
|
|
|
+
|
|
|
+ def save(self, *args, **kwargs):
|
|
|
+ is_new = self._state.adding
|
|
|
+ # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories
|
|
|
+ # Call the Django Model.save() directly instead
|
|
|
+ models.Model.save(self, *args, **kwargs)
|
|
|
+
|
|
|
+ if is_new:
|
|
|
+ from archivebox.misc.logging_util import log_worker_event
|
|
|
+ log_worker_event(
|
|
|
+ worker_type='DB',
|
|
|
+ event='Created ArchiveResult',
|
|
|
+ indent_level=3,
|
|
|
+ plugin=self.plugin,
|
|
|
+ metadata={
|
|
|
+ 'id': str(self.id),
|
|
|
+ 'snapshot_id': str(self.snapshot_id),
|
|
|
+ 'snapshot_url': str(self.snapshot.url)[:64],
|
|
|
+ 'status': self.status,
|
|
|
+ },
|
|
|
+ )
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def snapshot_dir(self):
|
|
|
+ return Path(self.snapshot.output_dir)
|
|
|
+
|
|
|
+ @cached_property
|
|
|
+ def url(self):
|
|
|
+ return self.snapshot.url
|
|
|
+
|
|
|
+ @property
|
|
|
+ def api_url(self) -> str:
|
|
|
+ return reverse_lazy('api-1:get_archiveresult', args=[self.id])
|
|
|
+
|
|
|
+ def get_absolute_url(self):
|
|
|
+ return f'/{self.snapshot.archive_path}/{self.plugin}'
|
|
|
+
|
|
|
+ @property
|
|
|
+ def plugin_module(self) -> Any | None:
|
|
|
+ # Hook scripts are now used instead of Python plugin modules
|
|
|
+ # The plugin name maps to hooks in archivebox/plugins/{plugin}/
|
|
|
+ return None
|
|
|
+
|
|
|
+ def output_exists(self) -> bool:
|
|
|
+ return os.path.exists(Path(self.snapshot_dir) / self.plugin)
|
|
|
+
|
|
|
+ def embed_path(self) -> Optional[str]:
|
|
|
+ """
|
|
|
+ Get the relative path to the embeddable output file for this result.
|
|
|
+
|
|
|
+ Returns the first file from output_files if set, otherwise tries to
|
|
|
+ find a reasonable default based on the plugin type.
|
|
|
+ """
|
|
|
+ # Check output_files dict for primary output
|
|
|
+ if self.output_files:
|
|
|
+ # Return first file from output_files (dict preserves insertion order)
|
|
|
+ first_file = next(iter(self.output_files.keys()), None)
|
|
|
+ if first_file:
|
|
|
+ return f'{self.plugin}/{first_file}'
|
|
|
+
|
|
|
+ # Fallback: check output_str if it looks like a file path
|
|
|
+ if self.output_str and ('/' in self.output_str or '.' in self.output_str):
|
|
|
+ return self.output_str
|
|
|
+
|
|
|
+ # Try to find output file based on plugin's canonical output path
|
|
|
+ canonical = self.snapshot.canonical_outputs()
|
|
|
+ plugin_key = f'{self.plugin}_path'
|
|
|
+ if plugin_key in canonical:
|
|
|
+ return canonical[plugin_key]
|
|
|
+
|
|
|
+ # Fallback to plugin directory
|
|
|
+ return f'{self.plugin}/'
|
|
|
+
|
|
|
+ def create_output_dir(self):
|
|
|
+ output_dir = Path(self.snapshot_dir) / self.plugin
|
|
|
+ output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
+ return output_dir
|
|
|
+
|
|
|
+ @property
|
|
|
+ def output_dir_name(self) -> str:
|
|
|
+ return self.plugin
|
|
|
+
|
|
|
+ @property
|
|
|
+ def output_dir_parent(self) -> str:
|
|
|
+ return str(self.snapshot.OUTPUT_DIR.relative_to(CONSTANTS.DATA_DIR))
|
|
|
+
|
|
|
+ def save_search_index(self):
|
|
|
+ pass
|
|
|
+
|
|
|
+ def cascade_health_update(self, success: bool):
|
|
|
+ """Update health stats for self, parent Snapshot, and grandparent Crawl (if present)."""
|
|
|
+ self.increment_health_stats(success)
|
|
|
+ self.snapshot.increment_health_stats(success)
|
|
|
+ if self.snapshot.crawl_id:
|
|
|
+ self.snapshot.crawl.increment_health_stats(success)
|
|
|
+
|
|
|
+ def run(self):
|
|
|
+ """
|
|
|
+ Execute this ArchiveResult's hook and update status.
|
|
|
+
|
|
|
+ If self.hook_name is set, runs only that specific hook.
|
|
|
+ If self.hook_name is empty, discovers and runs all hooks for self.plugin (backwards compat).
|
|
|
+
|
|
|
+ Updates status/output fields, queues discovered URLs, and triggers indexing.
|
|
|
+ """
|
|
|
+ from django.utils import timezone
|
|
|
+ from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, run_hook, is_background_hook
|
|
|
+ from archivebox.config.configset import get_config
|
|
|
+
|
|
|
+ # Get merged config with proper context
|
|
|
+ config = get_config(
|
|
|
+ crawl=self.snapshot.crawl if self.snapshot.crawl else None,
|
|
|
+ snapshot=self.snapshot,
|
|
|
+ )
|
|
|
+
|
|
|
+ # Determine which hook(s) to run
|
|
|
+ hooks = []
|
|
|
+
|
|
|
+ if self.hook_name:
|
|
|
+ # SPECIFIC HOOK MODE: Find the specific hook by name
|
|
|
+ for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
|
|
|
+ if not base_dir.exists():
|
|
|
+ continue
|
|
|
+ plugin_dir = base_dir / self.plugin
|
|
|
+ if plugin_dir.exists():
|
|
|
+ hook_path = plugin_dir / self.hook_name
|
|
|
+ if hook_path.exists():
|
|
|
+ hooks.append(hook_path)
|
|
|
+ break
|
|
|
+ else:
|
|
|
+ # LEGACY MODE: Discover all hooks for this plugin (backwards compatibility)
|
|
|
+ for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
|
|
|
+ if not base_dir.exists():
|
|
|
+ continue
|
|
|
+ plugin_dir = base_dir / self.plugin
|
|
|
+ if plugin_dir.exists():
|
|
|
+ matches = list(plugin_dir.glob('on_Snapshot__*.*'))
|
|
|
+ if matches:
|
|
|
+ hooks.extend(sorted(matches))
|
|
|
+
|
|
|
+ if not hooks:
|
|
|
+ self.status = self.StatusChoices.FAILED
|
|
|
+ if self.hook_name:
|
|
|
+ self.output_str = f'Hook not found: {self.plugin}/{self.hook_name}'
|
|
|
+ else:
|
|
|
+ self.output_str = f'No hooks found for plugin: {self.plugin}'
|
|
|
+ self.retry_at = None
|
|
|
+ self.save()
|
|
|
+ return
|
|
|
+
|
|
|
+ # Output directory is plugin_dir for the hook output
|
|
|
+ plugin_dir = Path(self.snapshot.output_dir) / self.plugin
|
|
|
+
|
|
|
+ start_ts = timezone.now()
|
|
|
+ is_bg_hook = False
|
|
|
+
|
|
|
+ for hook in hooks:
|
|
|
+ # Check if this is a background hook
|
|
|
+ is_bg_hook = is_background_hook(hook.name)
|
|
|
+
|
|
|
+ result = run_hook(
|
|
|
+ hook,
|
|
|
+ output_dir=plugin_dir,
|
|
|
+ config=config,
|
|
|
+ url=self.snapshot.url,
|
|
|
+ snapshot_id=str(self.snapshot.id),
|
|
|
+ crawl_id=str(self.snapshot.crawl.id) if self.snapshot.crawl else None,
|
|
|
+ depth=self.snapshot.depth,
|
|
|
+ )
|
|
|
+
|
|
|
+ # Background hooks return None
|
|
|
+ if result is None:
|
|
|
+ is_bg_hook = True
|
|
|
+
|
|
|
+ # Update status based on hook execution
|
|
|
+ if is_bg_hook:
|
|
|
+ # BACKGROUND HOOK - still running, return immediately
|
|
|
+ # Status stays STARTED, will be finalized by Snapshot.cleanup()
|
|
|
+ self.status = self.StatusChoices.STARTED
|
|
|
+ self.start_ts = start_ts
|
|
|
+ self.pwd = str(plugin_dir)
|
|
|
+ self.save()
|
|
|
+ return
|
|
|
+
|
|
|
+ # FOREGROUND HOOK - completed, update from filesystem
|
|
|
+ self.start_ts = start_ts
|
|
|
+ self.pwd = str(plugin_dir)
|
|
|
+ self.update_from_output()
|
|
|
+
|
|
|
+ # Clean up empty output directory if no files were created
|
|
|
+ if plugin_dir.exists() and not self.output_files:
|
|
|
+ try:
|
|
|
+ if not any(plugin_dir.iterdir()):
|
|
|
+ plugin_dir.rmdir()
|
|
|
+ except (OSError, RuntimeError):
|
|
|
+ pass
|
|
|
+
|
|
|
+ def update_from_output(self):
|
|
|
+ """
|
|
|
+ Update this ArchiveResult from filesystem logs and output files.
|
|
|
+
|
|
|
+ Used for:
|
|
|
+ - Foreground hooks that completed (called from ArchiveResult.run())
|
|
|
+ - Background hooks that completed (called from Snapshot.cleanup())
|
|
|
+
|
|
|
+ Updates:
|
|
|
+ - status, output_str, output_json from ArchiveResult JSONL record
|
|
|
+ - output_files, output_size, output_mimetypes by walking filesystem
|
|
|
+ - end_ts, retry_at, cmd, cmd_version, binary FK
|
|
|
+ - Processes side-effect records (Snapshot, Tag, etc.) via process_hook_records()
|
|
|
+ """
|
|
|
+ import json
|
|
|
+ import mimetypes
|
|
|
+ from collections import defaultdict
|
|
|
+ from pathlib import Path
|
|
|
+ from django.utils import timezone
|
|
|
+ from archivebox.hooks import process_hook_records
|
|
|
+
|
|
|
+ plugin_dir = Path(self.pwd) if self.pwd else None
|
|
|
+ if not plugin_dir or not plugin_dir.exists():
|
|
|
+ self.status = self.StatusChoices.FAILED
|
|
|
+ self.output_str = 'Output directory not found'
|
|
|
+ self.end_ts = timezone.now()
|
|
|
+ self.retry_at = None
|
|
|
+ self.save()
|
|
|
+ return
|
|
|
+
|
|
|
+ # Read and parse JSONL output from stdout.log
|
|
|
+ stdout_file = plugin_dir / 'stdout.log'
|
|
|
+ stdout = stdout_file.read_text() if stdout_file.exists() else ''
|
|
|
+
|
|
|
+ records = []
|
|
|
+ for line in stdout.splitlines():
|
|
|
+ if line.strip() and line.strip().startswith('{'):
|
|
|
+ try:
|
|
|
+ records.append(json.loads(line))
|
|
|
+ except json.JSONDecodeError:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # Find ArchiveResult record and update status/output from it
|
|
|
+ ar_records = [r for r in records if r.get('type') == 'ArchiveResult']
|
|
|
+ if ar_records:
|
|
|
+ hook_data = ar_records[0]
|
|
|
+
|
|
|
+ # Update status
|
|
|
+ status_map = {
|
|
|
+ 'succeeded': self.StatusChoices.SUCCEEDED,
|
|
|
+ 'failed': self.StatusChoices.FAILED,
|
|
|
+ 'skipped': self.StatusChoices.SKIPPED,
|
|
|
+ }
|
|
|
+ self.status = status_map.get(hook_data.get('status', 'failed'), self.StatusChoices.FAILED)
|
|
|
+
|
|
|
+ # Update output fields
|
|
|
+ self.output_str = hook_data.get('output_str') or hook_data.get('output') or ''
|
|
|
+ self.output_json = hook_data.get('output_json')
|
|
|
+
|
|
|
+ # Update cmd fields
|
|
|
+ if hook_data.get('cmd'):
|
|
|
+ self.cmd = hook_data['cmd']
|
|
|
+ self._set_binary_from_cmd(hook_data['cmd'])
|
|
|
+ if hook_data.get('cmd_version'):
|
|
|
+ self.cmd_version = hook_data['cmd_version'][:128]
|
|
|
+ else:
|
|
|
+ # No ArchiveResult record = failed
|
|
|
+ self.status = self.StatusChoices.FAILED
|
|
|
+ self.output_str = 'Hook did not output ArchiveResult record'
|
|
|
+
|
|
|
+ # Walk filesystem and populate output_files, output_size, output_mimetypes
|
|
|
+ exclude_names = {'stdout.log', 'stderr.log', 'hook.pid', 'listener.pid'}
|
|
|
+ mime_sizes = defaultdict(int)
|
|
|
+ total_size = 0
|
|
|
+ output_files = {}
|
|
|
+
|
|
|
+ for file_path in plugin_dir.rglob('*'):
|
|
|
+ if not file_path.is_file():
|
|
|
+ continue
|
|
|
+ if file_path.name in exclude_names:
|
|
|
+ continue
|
|
|
+
|
|
|
+ try:
|
|
|
+ stat = file_path.stat()
|
|
|
+ mime_type, _ = mimetypes.guess_type(str(file_path))
|
|
|
+ mime_type = mime_type or 'application/octet-stream'
|
|
|
+
|
|
|
+ relative_path = str(file_path.relative_to(plugin_dir))
|
|
|
+ output_files[relative_path] = {}
|
|
|
+ mime_sizes[mime_type] += stat.st_size
|
|
|
+ total_size += stat.st_size
|
|
|
+ except (OSError, IOError):
|
|
|
+ continue
|
|
|
+
|
|
|
+ self.output_files = output_files
|
|
|
+ self.output_size = total_size
|
|
|
+ sorted_mimes = sorted(mime_sizes.items(), key=lambda x: x[1], reverse=True)
|
|
|
+ self.output_mimetypes = ','.join(mime for mime, _ in sorted_mimes)
|
|
|
+
|
|
|
+ # Update timestamps
|
|
|
+ self.end_ts = timezone.now()
|
|
|
+ self.retry_at = None
|
|
|
+
|
|
|
+ self.save()
|
|
|
+
|
|
|
+ # Process side-effect records (filter Snapshots for depth/URL)
|
|
|
+ filtered_records = []
|
|
|
+ for record in records:
|
|
|
+ record_type = record.get('type')
|
|
|
+
|
|
|
+ # Skip ArchiveResult records (already processed above)
|
|
|
+ if record_type == 'ArchiveResult':
|
|
|
+ continue
|
|
|
+
|
|
|
+ # Filter Snapshot records for depth/URL constraints
|
|
|
+ if record_type == 'Snapshot':
|
|
|
+ if not self.snapshot.crawl:
|
|
|
+ continue
|
|
|
+
|
|
|
+ url = record.get('url')
|
|
|
+ if not url:
|
|
|
+ continue
|
|
|
+
|
|
|
+ depth = record.get('depth', self.snapshot.depth + 1)
|
|
|
+ if depth > self.snapshot.crawl.max_depth:
|
|
|
+ continue
|
|
|
+
|
|
|
+ if not self._url_passes_filters(url):
|
|
|
+ continue
|
|
|
+
|
|
|
+ filtered_records.append(record)
|
|
|
+
|
|
|
+ # Process filtered records with unified dispatcher
|
|
|
+ overrides = {
|
|
|
+ 'snapshot': self.snapshot,
|
|
|
+ 'crawl': self.snapshot.crawl,
|
|
|
+ 'created_by_id': self.snapshot.crawl.created_by_id,
|
|
|
+ }
|
|
|
+ process_hook_records(filtered_records, overrides=overrides)
|
|
|
+
|
|
|
+ # Cleanup PID files and empty logs
|
|
|
+ pid_file = plugin_dir / 'hook.pid'
|
|
|
+ pid_file.unlink(missing_ok=True)
|
|
|
+ stderr_file = plugin_dir / 'stderr.log'
|
|
|
+ if stdout_file.exists() and stdout_file.stat().st_size == 0:
|
|
|
+ stdout_file.unlink()
|
|
|
+ if stderr_file.exists() and stderr_file.stat().st_size == 0:
|
|
|
+ stderr_file.unlink()
|
|
|
+
|
|
|
+ def _set_binary_from_cmd(self, cmd: list) -> None:
|
|
|
+ """
|
|
|
+ Find Binary for command and set binary FK.
|
|
|
+
|
|
|
+ Tries matching by absolute path first, then by binary name.
|
|
|
+ Only matches binaries on the current machine.
|
|
|
+ """
|
|
|
+ if not cmd:
|
|
|
+ return
|
|
|
+
|
|
|
+ from archivebox.machine.models import Machine
|
|
|
+
|
|
|
+ bin_path_or_name = cmd[0] if isinstance(cmd, list) else cmd
|
|
|
+ machine = Machine.current()
|
|
|
+
|
|
|
+ # Try matching by absolute path first
|
|
|
+ binary = Binary.objects.filter(
|
|
|
+ abspath=bin_path_or_name,
|
|
|
+ machine=machine
|
|
|
+ ).first()
|
|
|
+
|
|
|
+ if binary:
|
|
|
+ self.binary = binary
|
|
|
+ return
|
|
|
+
|
|
|
+ # Fallback: match by binary name
|
|
|
+ bin_name = Path(bin_path_or_name).name
|
|
|
+ binary = Binary.objects.filter(
|
|
|
+ name=bin_name,
|
|
|
+ machine=machine
|
|
|
+ ).first()
|
|
|
+
|
|
|
+ if binary:
|
|
|
+ self.binary = binary
|
|
|
+
|
|
|
+ def _url_passes_filters(self, url: str) -> bool:
|
|
|
+ """Check if URL passes URL_ALLOWLIST and URL_DENYLIST config filters.
|
|
|
+
|
|
|
+ Uses proper config hierarchy: defaults -> file -> env -> machine -> user -> crawl -> snapshot
|
|
|
+ """
|
|
|
+ import re
|
|
|
+ from archivebox.config.configset import get_config
|
|
|
+
|
|
|
+ # Get merged config with proper hierarchy
|
|
|
+ config = get_config(
|
|
|
+ user=self.snapshot.crawl.created_by if self.snapshot else None,
|
|
|
+ crawl=self.snapshot.crawl if self.snapshot else None,
|
|
|
+ snapshot=self.snapshot,
|
|
|
+ )
|
|
|
+
|
|
|
+ # Get allowlist/denylist (can be string or list)
|
|
|
+ allowlist_raw = config.get('URL_ALLOWLIST', '')
|
|
|
+ denylist_raw = config.get('URL_DENYLIST', '')
|
|
|
+
|
|
|
+ # Normalize to list of patterns
|
|
|
+ def to_pattern_list(value):
|
|
|
+ if isinstance(value, list):
|
|
|
+ return value
|
|
|
+ if isinstance(value, str):
|
|
|
+ return [p.strip() for p in value.split(',') if p.strip()]
|
|
|
+ return []
|
|
|
+
|
|
|
+ allowlist = to_pattern_list(allowlist_raw)
|
|
|
+ denylist = to_pattern_list(denylist_raw)
|
|
|
+
|
|
|
+ # Denylist takes precedence
|
|
|
+ if denylist:
|
|
|
+ for pattern in denylist:
|
|
|
+ try:
|
|
|
+ if re.search(pattern, url):
|
|
|
+ return False
|
|
|
+ except re.error:
|
|
|
+ continue # Skip invalid regex patterns
|
|
|
+
|
|
|
+ # If allowlist exists, URL must match at least one pattern
|
|
|
+ if allowlist:
|
|
|
+ for pattern in allowlist:
|
|
|
+ try:
|
|
|
+ if re.search(pattern, url):
|
|
|
+ return True
|
|
|
+ except re.error:
|
|
|
+ continue # Skip invalid regex patterns
|
|
|
+ return False # No allowlist patterns matched
|
|
|
+
|
|
|
+ return True # No filters or passed filters
|
|
|
+
|
|
|
+ @property
|
|
|
+ def output_dir(self) -> Path:
|
|
|
+ """Get the output directory for this plugin's results."""
|
|
|
+ return Path(self.snapshot.output_dir) / self.plugin
|
|
|
+
|
|
|
+ def is_background_hook(self) -> bool:
|
|
|
+ """Check if this ArchiveResult is for a background hook."""
|
|
|
+ plugin_dir = Path(self.pwd) if self.pwd else None
|
|
|
+ if not plugin_dir:
|
|
|
+ return False
|
|
|
+ pid_file = plugin_dir / 'hook.pid'
|
|
|
+ return pid_file.exists()
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# ArchiveResult State Machine
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+class ArchiveResultMachine(BaseStateMachine, strict_states=True):
|
|
|
+ """
|
|
|
+ State machine for managing ArchiveResult (single plugin execution) lifecycle.
|
|
|
+
|
|
|
+ Hook Lifecycle:
|
|
|
+ ┌─────────────────────────────────────────────────────────────┐
|
|
|
+ │ QUEUED State │
|
|
|
+ │ • Waiting for its turn to run │
|
|
|
+ └─────────────────────────────────────────────────────────────┘
|
|
|
+ ↓ tick() when can_start()
|
|
|
+ ┌─────────────────────────────────────────────────────────────┐
|
|
|
+ │ STARTED State → enter_started() │
|
|
|
+ │ 1. archiveresult.run() │
|
|
|
+ │ • Find specific hook by hook_name │
|
|
|
+ │ • run_hook(script, output_dir, ...) → subprocess │
|
|
|
+ │ │
|
|
|
+ │ 2a. FOREGROUND hook (returns HookResult): │
|
|
|
+ │ • update_from_output() immediately │
|
|
|
+ │ - Read stdout.log │
|
|
|
+ │ - Parse JSONL records │
|
|
|
+ │ - Extract 'ArchiveResult' record → update status │
|
|
|
+ │ - Walk output_dir → populate output_files │
|
|
|
+ │ - Call process_hook_records() for side effects │
|
|
|
+ │ │
|
|
|
+ │ 2b. BACKGROUND hook (returns None): │
|
|
|
+ │ • Status stays STARTED │
|
|
|
+ │ • Continues running in background │
|
|
|
+ │ • Killed by Snapshot.cleanup() when sealed │
|
|
|
+ └─────────────────────────────────────────────────────────────┘
|
|
|
+ ↓ tick() checks status
|
|
|
+ ┌─────────────────────────────────────────────────────────────┐
|
|
|
+ │ SUCCEEDED / FAILED / SKIPPED / BACKOFF │
|
|
|
+ │ • Set by hook's JSONL output during update_from_output() │
|
|
|
+ │ • Health stats incremented (num_uses_succeeded/failed) │
|
|
|
+ │ • Parent Snapshot health stats also updated │
|
|
|
+ └─────────────────────────────────────────────────────────────┘
|
|
|
+
|
|
|
+ https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
|
|
|
+ """
|
|
|
+
|
|
|
+ model_attr_name = 'archiveresult'
|
|
|
+
|
|
|
+ # States
|
|
|
+ queued = State(value=ArchiveResult.StatusChoices.QUEUED, initial=True)
|
|
|
+ started = State(value=ArchiveResult.StatusChoices.STARTED)
|
|
|
+ backoff = State(value=ArchiveResult.StatusChoices.BACKOFF)
|
|
|
+ succeeded = State(value=ArchiveResult.StatusChoices.SUCCEEDED, final=True)
|
|
|
+ failed = State(value=ArchiveResult.StatusChoices.FAILED, final=True)
|
|
|
+ skipped = State(value=ArchiveResult.StatusChoices.SKIPPED, final=True)
|
|
|
+
|
|
|
+ # Tick Event - transitions based on conditions
|
|
|
+ tick = (
|
|
|
+ queued.to.itself(unless='can_start') |
|
|
|
+ queued.to(started, cond='can_start') |
|
|
|
+ started.to.itself(unless='is_finished') |
|
|
|
+ started.to(succeeded, cond='is_succeeded') |
|
|
|
+ started.to(failed, cond='is_failed') |
|
|
|
+ started.to(skipped, cond='is_skipped') |
|
|
|
+ started.to(backoff, cond='is_backoff') |
|
|
|
+ backoff.to.itself(unless='can_start') |
|
|
|
+ backoff.to(started, cond='can_start') |
|
|
|
+ backoff.to(succeeded, cond='is_succeeded') |
|
|
|
+ backoff.to(failed, cond='is_failed') |
|
|
|
+ backoff.to(skipped, cond='is_skipped')
|
|
|
+ )
|
|
|
+
|
|
|
+ def can_start(self) -> bool:
|
|
|
+ can_start = bool(self.archiveresult.snapshot.url)
|
|
|
+ # Suppressed: queue waiting logs
|
|
|
+ return can_start
|
|
|
+
|
|
|
+ def is_succeeded(self) -> bool:
|
|
|
+ """Check if extractor plugin succeeded (status was set by run())."""
|
|
|
+ return self.archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED
|
|
|
+
|
|
|
+ def is_failed(self) -> bool:
|
|
|
+ """Check if extractor plugin failed (status was set by run())."""
|
|
|
+ return self.archiveresult.status == ArchiveResult.StatusChoices.FAILED
|
|
|
+
|
|
|
+ def is_skipped(self) -> bool:
|
|
|
+ """Check if extractor plugin was skipped (status was set by run())."""
|
|
|
+ return self.archiveresult.status == ArchiveResult.StatusChoices.SKIPPED
|
|
|
+
|
|
|
+ def is_backoff(self) -> bool:
|
|
|
+ """Check if we should backoff and retry later."""
|
|
|
+ # Backoff if status is still started (plugin didn't complete) and output_str is empty
|
|
|
+ return (
|
|
|
+ self.archiveresult.status == ArchiveResult.StatusChoices.STARTED and
|
|
|
+ not self.archiveresult.output_str
|
|
|
+ )
|
|
|
+
|
|
|
+ def is_finished(self) -> bool:
|
|
|
+ """Check if extraction has completed (success, failure, or skipped)."""
|
|
|
+ return self.archiveresult.status in (
|
|
|
+ ArchiveResult.StatusChoices.SUCCEEDED,
|
|
|
+ ArchiveResult.StatusChoices.FAILED,
|
|
|
+ ArchiveResult.StatusChoices.SKIPPED,
|
|
|
+ )
|
|
|
+
|
|
|
+ @queued.enter
|
|
|
+ def enter_queued(self):
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ self.archiveresult.update_and_requeue(
|
|
|
+ retry_at=timezone.now(),
|
|
|
+ status=ArchiveResult.StatusChoices.QUEUED,
|
|
|
+ start_ts=None,
|
|
|
+ ) # bump the snapshot's retry_at so they pickup any new changes
|
|
|
+
|
|
|
+ @started.enter
|
|
|
+ def enter_started(self):
|
|
|
+ from archivebox.machine.models import NetworkInterface
|
|
|
+
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ # Lock the object and mark start time
|
|
|
+ self.archiveresult.update_and_requeue(
|
|
|
+ retry_at=timezone.now() + timedelta(seconds=120), # 2 min timeout for plugin
|
|
|
+ status=ArchiveResult.StatusChoices.STARTED,
|
|
|
+ start_ts=timezone.now(),
|
|
|
+ iface=NetworkInterface.current(),
|
|
|
+ )
|
|
|
+
|
|
|
+ # Run the plugin - this updates status, output, timestamps, etc.
|
|
|
+ self.archiveresult.run()
|
|
|
+
|
|
|
+ # Save the updated result
|
|
|
+ self.archiveresult.save()
|
|
|
+
|
|
|
+ # Suppressed: plugin result logs (already logged by worker)
|
|
|
+
|
|
|
+ @backoff.enter
|
|
|
+ def enter_backoff(self):
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ self.archiveresult.update_and_requeue(
|
|
|
+ retry_at=timezone.now() + timedelta(seconds=60),
|
|
|
+ status=ArchiveResult.StatusChoices.BACKOFF,
|
|
|
+ end_ts=None,
|
|
|
+ # retries=F('retries') + 1, # F() equivalent to getattr(self.archiveresult, 'retries', 0) + 1,
|
|
|
+ )
|
|
|
+
|
|
|
+ @succeeded.enter
|
|
|
+ def enter_succeeded(self):
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ self.archiveresult.update_and_requeue(
|
|
|
+ retry_at=None,
|
|
|
+ status=ArchiveResult.StatusChoices.SUCCEEDED,
|
|
|
+ end_ts=timezone.now(),
|
|
|
+ # **self.archiveresult.get_output_dict(), # {output, output_json, stderr, stdout, returncode, errors, cmd_version, pwd, cmd, machine}
|
|
|
+ )
|
|
|
+ self.archiveresult.save()
|
|
|
+
|
|
|
+ # Update health stats for ArchiveResult, Snapshot, and Crawl cascade
|
|
|
+ self.archiveresult.cascade_health_update(success=True)
|
|
|
+
|
|
|
+ @failed.enter
|
|
|
+ def enter_failed(self):
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ self.archiveresult.update_and_requeue(
|
|
|
+ retry_at=None,
|
|
|
+ status=ArchiveResult.StatusChoices.FAILED,
|
|
|
+ end_ts=timezone.now(),
|
|
|
+ )
|
|
|
+
|
|
|
+ # Update health stats for ArchiveResult, Snapshot, and Crawl cascade
|
|
|
+ self.archiveresult.cascade_health_update(success=False)
|
|
|
+
|
|
|
+ @skipped.enter
|
|
|
+ def enter_skipped(self):
|
|
|
+ # Suppressed: state transition logs
|
|
|
+ self.archiveresult.update_and_requeue(
|
|
|
+ retry_at=None,
|
|
|
+ status=ArchiveResult.StatusChoices.SKIPPED,
|
|
|
+ end_ts=timezone.now(),
|
|
|
+ )
|
|
|
+
|
|
|
+ def after_transition(self, event: str, source: State, target: State):
|
|
|
+ # print(f"after '{event}' from '{source.id}' to '{target.id}'")
|
|
|
+ self.archiveresult.snapshot.update_and_requeue() # bump snapshot retry time so it picks up all the new changes
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# State Machine Registration
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+# Manually register state machines with python-statemachine registry
|
|
|
+# (normally auto-discovered from statemachines.py, but we define them here for clarity)
|
|
|
+registry.register(SnapshotMachine)
|
|
|
+registry.register(ArchiveResultMachine)
|
Nick Sweeting