BasePlugin system expanded and registration system improved

archivebox/builtin_plugins/base/apps.py

@@ -1,83 +0,0 @@
-import sys
-import inspect
-from typing import List, Dict, Any, Optional
-from pathlib import Path
-
-import django
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-from plugantic.replayers import Replayer
-
-
-class PythonBinary(Binary):
-    name: BinName = 'python'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides: Dict[str, Any] = {
-        'env': {
-            'subdeps': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
-            'abspath': \
-                lambda: sys.executable,
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
-        },
-    }
-
-class SqliteBinary(Binary):
-    name: BinName = 'sqlite'
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: Path(inspect.getfile(sqlite3)),
-            'version': \
-                lambda: SemVer(sqlite3.version),
-        },
-    }
-
-
-class DjangoBinary(Binary):
-    name: BinName = 'django'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(django),
-            'version': \
-                lambda: django.VERSION[:3],
-        },
-    }
-
-
-class BasicReplayer(Replayer):
-    name: str = 'basic'
-
-
-class BasePlugin(Plugin):
-    name: str = 'base'
-    configs: List[SerializeAsAny[ConfigSet]] = []
-    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = []
-    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
-
-
-PLUGINS = [BasePlugin()]
-
-
-class BaseConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'builtin_plugins.base'

archivebox/builtin_plugins/npm/apps.py

@@ -0,0 +1,66 @@
+__package__ = 'archivebox.builtin_plugins.npm'
+
+from pathlib import Path
+from typing import List, Dict, Optional
+from pydantic import InstanceOf, Field
+
+from django.apps import AppConfig
+from django.conf import settings
+
+from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_configset import ConfigSectionName
+
+from pkg.settings import env, apt, brew
+
+from ...config import CONFIG
+
+###################### Config ##########################
+
+
+class NpmDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_NPM: bool = True
+    NPM_BINARY: str = Field(default='npm')
+    NPM_ARGS: Optional[List[str]] = Field(default=None)
+    NPM_EXTRA_ARGS: List[str] = []
+    NPM_DEFAULT_ARGS: List[str] = []
+
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+class NpmProvider(NpmProvider, BaseBinProvider):
+    PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
+
+npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
+
+class NpmBinary(BaseBinary):
+    name: BinName = 'npm'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew]
+
+
+NPM_BINARY = NpmBinary()
+
+
+
+class NpmPlugin(BasePlugin):
+    name: str = 'builtin_plugins.npm'
+    app_label: str = 'npm'
+    verbose_name: str = 'NPM'
+
+    configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG]
+    binproviders: List[InstanceOf[BaseBinProvider]] = [npm]
+    binaries: List[InstanceOf[BaseBinary]] = [NPM_BINARY]
+
+
+PLUGIN = NpmPlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/pip/apps.py

@@ -0,0 +1,66 @@
+import sys
+from pathlib import Path
+from typing import List, Dict, Optional
+from pydantic import InstanceOf, Field
+
+from django.apps import AppConfig
+
+from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_configset import ConfigSectionName
+
+from pkg.settings import env, apt, brew
+
+
+###################### Config ##########################
+
+
+class PipDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_PIP: bool = True
+    PIP_BINARY: str = Field(default='pip')
+    PIP_ARGS: Optional[List[str]] = Field(default=None)
+    PIP_EXTRA_ARGS: List[str] = []
+    PIP_DEFAULT_ARGS: List[str] = []
+
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+class PipProvider(PipProvider, BaseBinProvider):
+    PATH: PATHStr = str(Path(sys.executable).parent)
+
+pip = PipProvider(PATH=str(Path(sys.executable).parent))
+
+
+class PipBinary(BaseBinary):
+    name: BinName = 'pip'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
+PIP_BINARY = PipBinary()
+
+
+
+
+
+
+
+
+class PipPlugin(BasePlugin):
+    name: str = 'builtin_plugins.pip'
+    app_label: str = 'pip'
+    verbose_name: str = 'PIP'
+
+    configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG]
+    binproviders: List[InstanceOf[BaseBinProvider]] = [pip]
+    binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY]
+
+
+PLUGIN = PipPlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,42 +1,31 @@
-from typing import List, Optional, Dict
 from pathlib import Path
+from typing import List, Dict, Optional
 
 from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
 
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
+# Depends on other PyPI/vendor packages:
+from pydantic import InstanceOf, Field
+from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
 from pydantic_pkgr.binprovider import bin_abspath
-from pydantic_pkgr.binary import BinProviderName, ProviderLookupDict
 
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
+# Depends on other Django apps:
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseExtractor, BaseReplayer
+from plugantic.base_configset import ConfigSectionName
 
+# Depends on Other Plugins:
 from pkg.settings import env
+from builtin_plugins.npm.apps import npm
 
 
 ###################### Config ##########################
 
-class SinglefileToggleConfig(ConfigSet):
+class SinglefileToggleConfigs(BaseConfigSet):
     section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
 
     SAVE_SINGLEFILE: bool = True
 
 
-class SinglefileDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    SINGLEFILE_BINARY: str = Field(default='wget')
-    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
-    SINGLEFILE_EXTRA_ARGS: List[str] = []
-    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class SinglefileOptionsConfig(ConfigSet):
+class SinglefileOptionsConfigs(BaseConfigSet):
     section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
 
     # loaded from shared config
@@ -47,67 +36,83 @@ class SinglefileOptionsConfig(ConfigSet):
     SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
 
 
+class SinglefileDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    SINGLEFILE_BINARY: str = Field(default='wget')
+    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class SinglefileConfigs(SinglefileToggleConfigs, SinglefileOptionsConfigs, SinglefileDependencyConfigs):
+    # section: ConfigSectionName = 'ALL_CONFIGS'
+    pass
 
-DEFAULT_CONFIG = {
+DEFAULT_GLOBAL_CONFIG = {
     'CHECK_SSL_VALIDITY': False,
     'SAVE_SINGLEFILE': True,
     'TIMEOUT': 120,
 }
 
-PLUGIN_CONFIG = [
-    SinglefileToggleConfig(**DEFAULT_CONFIG),
-    SinglefileDependencyConfig(**DEFAULT_CONFIG),
-    SinglefileOptionsConfig(**DEFAULT_CONFIG),
+SINGLEFILE_CONFIGS = [
+    SinglefileToggleConfigs(**DEFAULT_GLOBAL_CONFIG),
+    SinglefileDependencyConfigs(**DEFAULT_GLOBAL_CONFIG),
+    SinglefileOptionsConfigs(**DEFAULT_GLOBAL_CONFIG),
 ]
 
-###################### Binaries ############################
+
 
 min_version: str = "1.1.54"
 max_version: str = "2.0.0"
 
-class SinglefileBinary(Binary):
-    name: BinName = 'single-file'
-    providers_supported: List[BinProvider] = [NpmProvider()]
+def get_singlefile_abspath() -> Optional[Path]:
+    return 
 
 
+class SinglefileBinary(BaseBinary):
+    name: BinName = 'single-file'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, npm]
+
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={
-        'env': {
-            'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
-        },
-        'npm': {
-            # 'abspath': lambda: bin_abspath('single-file', PATH=NpmProvider().PATH) or bin_abspath('single-file', PATH=env.PATH),
-            'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
-        },
+        # 'env': {
+        #     'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
+        # },
+        # 'npm': {
+        #     'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH),
+        #     'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
+        # },
     }
 
+SINGLEFILE_BINARY = SinglefileBinary()
 
-###################### Extractors ##########################
+PLUGIN_BINARIES = [SINGLEFILE_BINARY]
 
-class SinglefileExtractor(Extractor):
-    name: ExtractorName = 'singlefile'
-    binary: Binary = SinglefileBinary()
+class SinglefileExtractor(BaseExtractor):
+    name: str = 'singlefile'
+    binary: BinName = SINGLEFILE_BINARY.name
 
     def get_output_path(self, snapshot) -> Path:
         return Path(snapshot.link_dir) / 'singlefile.html'
 
 
-###################### Plugins #############################
+SINGLEFILE_BINARY = SinglefileBinary()
+SINGLEFILE_EXTRACTOR = SinglefileExtractor()
 
+class SinglefilePlugin(BasePlugin):
+    name: str = 'builtin_plugins.singlefile'
+    app_label: str ='singlefile'
+    verbose_name: str = 'SingleFile'
 
-class SinglefilePlugin(Plugin):
-    name: str = 'singlefile'
-    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
-    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
-
-PLUGINS = [SinglefilePlugin()]
+    configs: List[InstanceOf[BaseConfigSet]] = SINGLEFILE_CONFIGS
+    binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY]
+    extractors: List[InstanceOf[BaseExtractor]] = [SINGLEFILE_EXTRACTOR]
 
-###################### Django Apps #########################
 
-class SinglefileConfig(AppConfig):
-    name = 'builtin_plugins.singlefile'
-    verbose_name = 'SingleFile'
 
-    def ready(self):
-        pass
-        # print('Loaded singlefile plugin')
+PLUGIN = SinglefilePlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/singlefile/config.yaml

@@ -1,66 +0,0 @@
-name: singlefile
-plugin_version: '0.0.1'
-plugin_spec: '0.0.1'
-
-binaries:
-    singlefile:
-        providers:
-            - env
-            - npm
-
-commands:
-    - singlefile.exec
-    - singlefile.extract
-    - singlefile.should_extract
-    - singlefile.get_output_path
-
-extractors:
-    singlefile:
-        binary: singlefile
-        test: singlefile.should_extract
-        extract: singlefile.extract
-        output_files:
-            - singlefile.html
-
-configs:
-    ARCHIVE_METHOD_TOGGLES:
-        SAVE_SINGLEFILE:
-            type: bool
-            default: true
-
-    DEPENDENCY_CONFIG:
-        SINGLEFILE_BINARY:
-            type: str
-            default: wget
-        SINGLEFILE_ARGS:
-            type: Optional[List[str]]
-            default: null
-        SINGLEFILE_EXTRA_ARGS:
-            type: List[str]
-            default: []
-        SINGLEFILE_DEFAULT_ARGS:
-            type: List[str]
-            default: 
-            - "--timeout={TIMEOUT-10}"
-
-    ARCHIVE_METHOD_OPTIONS:
-        SINGLEFILE_USER_AGENT:
-            type: str
-            default: ""
-            alias: USER_AGENT
-        SINGLEFILE_TIMEOUT:
-            type: int
-            default: 60
-            alias: TIMEOUT
-        SINGLEFILE_CHECK_SSL_VALIDITY:
-            type: bool
-            default: true
-            alias: CHECK_SSL_VALIDITY
-        SINGLEFILE_RESTRICT_FILE_NAMES:
-            type: str
-            default: windows
-            alias: RESTRICT_FILE_NAMES
-        SINGLEFILE_COOKIES_FILE:
-            type: Optional[Path]
-            default: null
-            alias: COOKIES_FILE

archivebox/builtin_plugins/singlefile/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/systempython/apps.py

@@ -0,0 +1,116 @@
+__package__ = 'archivebox.builtin_plugins.systempython'
+
+import os
+import sys
+import inspect
+from typing import List, Dict, Any, Callable, ClassVar
+from pathlib import Path
+
+import django
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+from django.utils.functional import classproperty
+from django.db.backends.sqlite3.base import Database as sqlite3
+from django.core.checks import Tags, Error, register
+
+from pydantic import InstanceOf, Field
+
+from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider, BaseExtractor, BaseReplayer
+from plugantic.base_check import BaseCheck
+
+from pkg.settings import env, apt, brew
+
+from builtin_plugins.pip.apps import pip
+
+class PythonBinary(BaseBinary):
+    name: BinName = 'python'
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
+        'apt': {
+            'subdeps': \
+                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+class SqliteBinary(BaseBinary):
+    name: BinName = 'sqlite'
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: Path(inspect.getfile(sqlite3)),
+            'version': \
+                lambda: SemVer(sqlite3.version),
+        },
+    }
+
+
+class DjangoBinary(BaseBinary):
+    name: BinName = 'django'
+
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+
+class BasicReplayer(BaseReplayer):
+    name: str = 'basic'
+
+
+
+
+class CheckUserIsNotRoot(BaseCheck):
+    label: str = 'CheckUserIsNotRoot'
+    tag = Tags.database
+
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        errors = []
+        if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0:
+            errors.append(
+                Error(
+                    "Cannot run as root!",
+                    id="core.S001",
+                    hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).',
+                )
+            )
+        logger.debug('[√] UID is not root')
+        return errors
+
+
+
+class SystemPythonPlugin(BasePlugin):
+    name: str = 'builtin_plugins.systempython'
+    app_label: str = 'systempython'
+    verbose_name: str = 'System Python'
+
+    configs: List[InstanceOf[BaseConfigSet]] = []
+    binaries: List[InstanceOf[BaseBinary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
+    extractors: List[InstanceOf[BaseExtractor]] = []
+    replayers: List[InstanceOf[BaseReplayer]] = [BasicReplayer()]
+    checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()]
+
+
+PLUGIN = SystemPythonPlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# PARSERS = PLUGIN.parsers
+# DAEMONS = PLUGIN.daemons
+# MODELS = PLUGIN.models
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/ytdlp/apps.py

@@ -0,0 +1,48 @@
+import sys
+from pathlib import Path
+from typing import List, Dict, Optional
+from pydantic import InstanceOf, Field
+
+from django.apps import AppConfig
+
+from pydantic_pkgr import BinProvider, BinName, PATHStr
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_configset import ConfigSectionName
+
+from pkg.settings import env, apt, brew
+
+from builtin_plugins.pip.apps import pip
+
+###################### Config ##########################
+
+
+class YtdlpDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_YTDLP: bool = True
+
+    YTDLP_BINARY: str = Field(default='yt-dlp')
+
+DEFAULT_GLOBAL_CONFIG = {}
+YTDLP_CONFIG = YtdlpDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+
+class YtdlpBinary(BaseBinary):
+    name: BinName = YTDLP_CONFIG.YTDLP_BINARY
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
+
+YTDLP_BINARY = YtdlpBinary()
+
+
+class YtdlpPlugin(BasePlugin):
+    name: str = 'builtin_plugins.ytdlp'
+    app_label: str = 'ytdlp'
+    verbose_name: str = 'YTDLP'
+
+    configs: List[InstanceOf[BaseConfigSet]] = [YTDLP_CONFIG]
+    binaries: List[InstanceOf[BaseBinary]] = [YTDLP_BINARY]
+
+
+PLUGIN = YtdlpPlugin()
+DJANGO_APP = PLUGIN.AppConfig

archivebox/core/settings.py

@@ -19,6 +19,46 @@ IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
 IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
 
+
+################################################################################
+### ArchiveBox Plugin Settings
+################################################################################
+
+BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins'  # /app/archivebox/builtin_plugins
+USERDATA_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins'     # /data/user_plugins
+
+def find_plugins_in_dir(plugins_dir, prefix: str) -> Dict[str, Path]:
+    return {
+        f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent
+        for plugin_entrypoint in sorted(plugins_dir.glob('*/apps.py'))
+    }
+
+INSTALLED_PLUGINS = {
+    **find_plugins_in_dir(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'),
+    **find_plugins_in_dir(USERDATA_PLUGINS_DIR, prefix='user_plugins'),
+}
+
+### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup)
+PLUGINS = AttrDict({})
+
+CONFIGS = AttrDict({})
+BINPROVIDERS = AttrDict({})
+BINARIES = AttrDict({})
+EXTRACTORS = AttrDict({})
+REPLAYERS = AttrDict({})
+CHECKS = AttrDict({})
+ADMINDATAVIEWS = AttrDict({})
+
+PLUGIN_KEYS = AttrDict({
+    'CONFIGS': CONFIGS,
+    'BINPROVIDERS': BINPROVIDERS,
+    'BINARIES': BINARIES,
+    'EXTRACTORS': EXTRACTORS,
+    'REPLAYERS': REPLAYERS,
+    'CHECKS': CHECKS,
+    'ADMINDATAVIEWS': ADMINDATAVIEWS,
+})
+
 ################################################################################
 ### Django Core Settings
 ################################################################################
@@ -35,52 +75,35 @@ APPEND_SLASH = True
 DEBUG = CONFIG.DEBUG or ('--debug' in sys.argv)
 
 
-BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins'
-USER_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins'
-
-def find_plugins(plugins_dir, prefix: str) -> Dict[str, Any]:
-    plugins = {
-        f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent
-        for plugin_entrypoint in plugins_dir.glob('*/apps.py')
-    }
-    # print(f'Found {prefix} plugins:\n', '\n    '.join(plugins.keys()))
-    return plugins
-
-INSTALLED_PLUGINS = {
-    **find_plugins(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'),
-    **find_plugins(USER_PLUGINS_DIR, prefix='user_plugins'),
-}
-
-
 INSTALLED_APPS = [
+    # Django default apps
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.admin',
-    'django_jsonform',
-    
-    'signal_webhooks',
-    'abid_utils',
-    'plugantic',
-    'core',
-    'api',
-    'pkg',
-
-    *INSTALLED_PLUGINS.keys(),
 
-    'admin_data_views',
-    'django_extensions',
+    # 3rd-party apps from PyPI
+    'django_jsonform',           # handles rendering Pydantic models to Django HTML widgets/forms
+    'signal_webhooks',           # handles REST API outbound webhooks
+    
+    # our own apps
+    'abid_utils',                # handles ABID ID creation, handling, and models
+    'plugantic',                 # ArchiveBox plugin API definition + finding/registering/calling interface
+    'core',                      # core django model with Snapshot, ArchiveResult, etc.
+    'api',                       # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
+    'pkg',                       # ArchiveBox runtime package management interface for subdependencies
+
+    # ArchiveBox plugins
+    *INSTALLED_PLUGINS.keys(),   # all plugin django-apps found in archivebox/builtin_plugins and data/user_plugins
+
+    # 3rd-party apps from PyPI that need to be loaded last
+    'admin_data_views',          # handles rendering some convenient automatic read-only views of data in Django admin
+    'django_extensions',         # provides Django Debug Toolbar (and other non-debug helpers)
 ]
 
 
-# For usage with https://www.jetadmin.io/integrations/django
-# INSTALLED_APPS += ['jet_django']
-# JET_PROJECT = 'archivebox'
-# JET_TOKEN = 'some-api-token-here'
-
-
 MIDDLEWARE = [
     'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',
@@ -371,8 +394,11 @@ LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
     'handlers': {
-        'console': {
-            'class': 'logging.StreamHandler',
+        "console": {
+            "level": "DEBUG",
+            "filters": [],
+            'formatter': 'simple',
+            "class": "logging.StreamHandler",
         },
         'logfile': {
             'level': 'ERROR',
@@ -380,14 +406,57 @@ LOGGING = {
             'filename': ERROR_LOG,
             'maxBytes': 1024 * 1024 * 25,  # 25 MB
             'backupCount': 10,
+            'formatter': 'verbose',
         },
+        # "mail_admins": {
+        #     "level": "ERROR",
+        #     "filters": ["require_debug_false"],
+        #     "class": "django.utils.log.AdminEmailHandler",
+        # },
     },
     'filters': {
         'noisyrequestsfilter': {
             '()': NoisyRequestsFilter,
-        }
+        },
+        "require_debug_false": {
+            "()": "django.utils.log.RequireDebugFalse",
+        },
+        "require_debug_true": {
+            "()": "django.utils.log.RequireDebugTrue",
+        },
+    },
+    'formatters': {
+        'verbose': {
+            'format': '{name} {levelname} {asctime} {module} {process:d} {thread:d} {message}',
+            'style': '{',
+        },
+        'simple': {
+            'format': '{name} {message}',
+            'style': '{',
+        },
+        "django.server": {
+            "()": "django.utils.log.ServerFormatter",
+            "format": "[{server_time}] {message}",
+            "style": "{",
+        },
     },
     'loggers': {
+        'api': {
+            'handlers': ['console', 'logfile'],
+            'level': 'DEBUG',
+        },
+        'checks': {
+            'handlers': ['console', 'logfile'],
+            'level': 'DEBUG',
+        },
+        'core': {
+            'handlers': ['console', 'logfile'],
+            'level': 'DEBUG',
+        },
+        'builtin_plugins': {
+            'handlers': ['console', 'logfile'],
+            'level': 'DEBUG',
+        },
         'django': {
             'handlers': ['console', 'logfile'],
             'level': 'INFO',
@@ -397,6 +466,8 @@ LOGGING = {
             'handlers': ['console', 'logfile'],
             'level': 'INFO',
             'filters': ['noisyrequestsfilter'],
+            'propagate': False,
+            "formatter": "django.server",
         }
     },
 }
@@ -541,3 +612,9 @@ if DEBUG_REQUESTS_TRACKER:
 # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.)
 DEBUG_LOGFIRE = False
 DEBUG_LOGFIRE = DEBUG_LOGFIRE and (Path(CONFIG.OUTPUT_DIR) / '.logfire').is_dir()
+
+
+# For usage with https://www.jetadmin.io/integrations/django
+# INSTALLED_APPS += ['jet_django']
+# JET_PROJECT = 'archivebox'
+# JET_TOKEN = 'some-api-token-here'

archivebox/main.py

@@ -318,7 +318,7 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=
         print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
     elif existing_index:
         # TODO: properly detect and print the existing version in current index as well
-        print('{green}[^] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI))
+        print('{green}[*] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI))
         print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
     else:
         if force:

archivebox/pkg/settings.py

@@ -10,77 +10,24 @@ import django
 from django.conf import settings
 from django.db.backends.sqlite3.base import Database as sqlite3
 
-from pydantic_pkgr import Binary, BinProvider, BrewProvider, EnvProvider, SemVer
+from pydantic_pkgr import Binary, BinProvider, BrewProvider, PipProvider, NpmProvider, AptProvider, EnvProvider, SemVer
 from pydantic_pkgr.binprovider import bin_abspath
 
 from ..config import NODE_BIN_PATH, bin_path
 
-env = EnvProvider(PATH=NODE_BIN_PATH + ':' + os.environ.get('PATH', '/bin'))
+apt = AptProvider()
+brew = BrewProvider()
+env = EnvProvider(PATH=os.environ.get('PATH', '/bin'))
 
+# Defined in their own plugins:
+#pip = PipProvider(PATH=str(Path(sys.executable).parent))
+#npm = NpmProvider(PATH=NODE_BIN_PATH)
 
 LOADED_DEPENDENCIES = {}
 
-for bin_key, dependency in settings.CONFIG.DEPENDENCIES.items():
-    # 'PYTHON_BINARY': {
-    #     'path': bin_path(config['PYTHON_BINARY']),
-    #     'version': config['PYTHON_VERSION'],
-    #     'hash': bin_hash(config['PYTHON_BINARY']),
-    #     'enabled': True,
-    #     'is_valid': bool(config['PYTHON_VERSION']),
-    # },
-    
-
-    bin_name = settings.CONFIG[bin_key]
-
-    if bin_name.endswith('django/__init__.py'):
-        binary_spec = Binary(name='django', providers=[env], provider_overrides={
-            'env': {
-                'abspath': lambda: Path(inspect.getfile(django)),
-                'version': lambda: SemVer('{}.{}.{} {} ({})'.format(*django.VERSION)),
-            }
-        })
-    elif bin_name.endswith('sqlite3/dbapi2.py'):
-        binary_spec = Binary(name='sqlite3', providers=[env], provider_overrides={
-            'env': {
-                'abspath': lambda: Path(inspect.getfile(sqlite3)),
-                'version': lambda: SemVer(sqlite3.version),
-            }
-        })
-    elif bin_name.endswith('archivebox'):
-        binary_spec = Binary(name='archivebox', providers=[env], provider_overrides={
-            'env': {
-                'abspath': lambda: shutil.which(str(Path('archivebox').expanduser())),
-                'version': lambda: settings.CONFIG.VERSION,
-            }
-        })
-    elif bin_name.endswith('postlight/parser/cli.js'):
-        binary_spec = Binary(name='postlight-parser', providers=[env], provider_overrides={
-            'env': {
-                'abspath': lambda: bin_path('postlight-parser'),
-                'version': lambda: SemVer('1.0.0'),
-            }
-        })
-    else:
-        binary_spec = Binary(name=bin_name, providers=[env])
-    
+for bin_name, binary_spec in settings.BINARIES.items():
     try:
-        binary = binary_spec.load()
+        settings.BINARIES[bin_name] = binary_spec.load()
     except Exception as e:
         # print(f"- ❌ Binary {bin_name} failed to load with error: {e}")
         continue
-
-    assert isinstance(binary.loaded_version, SemVer)
-
-    try:
-        assert str(binary.loaded_version) == dependency['version'], f"Expected {bin_name} version {dependency['version']}, got {binary.loaded_version}"
-        assert str(binary.loaded_respath) == str(bin_abspath(dependency['path']).resolve()), f"Expected {bin_name} abspath {bin_abspath(dependency['path']).resolve()}, got {binary.loaded_respath}"
-        assert binary.is_valid == dependency['is_valid'], f"Expected {bin_name} is_valid={dependency['is_valid']}, got {binary.is_valid}"
-    except Exception as e:
-        pass
-        # print(f"WARNING: Error loading {bin_name}: {e}")
-        # import ipdb; ipdb.set_trace()
-    
-    # print(f"- ✅ Binary {bin_name} loaded successfully")
-    LOADED_DEPENDENCIES[bin_key] = binary
-
-

archivebox/plugantic/__init__.py

@@ -1,16 +1,9 @@
 __package__ = 'archivebox.plugantic'
 
-from .binaries import Binary
-from .extractors import Extractor
-from .replayers import Replayer
-from .configs import ConfigSet
-from .plugins import Plugin
+from .base_plugin import BasePlugin
+from .base_configset import BaseConfigSet
+from .base_binary import BaseBinary
+from .base_extractor import BaseExtractor
+from .base_replayer import BaseReplayer
+from .base_check import BaseCheck
 
-# __all__ = [
-#     'BinProvider',
-#     'Binary',
-#     'Extractor',
-#     'Replayer',
-#     'ConfigSet',
-#     'Plugin',
-# ]

archivebox/plugantic/apps.py

@@ -1,6 +1,9 @@
+__package__ = 'archivebox.plugantic'
+
+import json
 import importlib
-from django.apps import AppConfig
 
+from django.apps import AppConfig
 
 class PluganticConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
@@ -8,10 +11,5 @@ class PluganticConfig(AppConfig):
 
     def ready(self) -> None:
         from django.conf import settings
-        from .plugins import PLUGINS
 
-        for plugin_name in settings.INSTALLED_PLUGINS.keys():
-            lib = importlib.import_module(f'{plugin_name}.apps')
-            if hasattr(lib, 'PLUGINS'):
-                for plugin_instance in lib.PLUGINS:
-                    PLUGINS.append(plugin_instance)
+        print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')

archivebox/plugantic/base_admindataview.py

@@ -0,0 +1,34 @@
+from typing import List, Type, Any, Dict
+
+from pydantic_core import core_schema
+from pydantic import GetCoreSchemaHandler, BaseModel
+
+from django.utils.functional import classproperty
+from django.core.checks import Warning, Tags, register
+
+class BaseAdminDataView(BaseModel):
+    name: str = 'NPM Installed Packages'
+    route: str = '/npm/installed/'
+    view: str = 'builtin_plugins.npm.admin.installed_list_view'
+    items: Dict[str, str] = {
+        "name": "installed_npm_pkg",
+        'route': '<str:key>/',
+        'view': 'builtin_plugins.npm.admin.installed_detail_view',
+    }
+
+    def as_route(self) -> Dict[str, str | Dict[str, str]]:
+        return {
+            'route': self.route,
+            'view': self.view,
+            'name': self.name,
+            'items': self.items,
+        }
+
+    def register(self, settings, parent_plugin=None):
+        """Regsiter AdminDataViews.as_route() in settings.ADMIN_DATA_VIEWS.URLS at runtime"""
+        self._plugin = parent_plugin                          # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
+
+        route = self.as_route()
+        if route not in settings.ADMIN_DATA_VIEWS.URLS:
+            settings.ADMIN_DATA_VIEWS.URLS += [route]         # append our route (update in place)
+

archivebox/plugantic/base_binary.py

@@ -0,0 +1,99 @@
+__package__ = 'archivebox.plugantic'
+
+import sys
+import inspect
+import importlib
+from pathlib import Path
+
+
+from typing import Any, Optional, Dict, List
+from typing_extensions import Self
+from subprocess import run, PIPE
+
+from pydantic import Field, InstanceOf
+from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
+from pydantic_pkgr.binprovider import HostBinPath
+
+import django
+from django.core.cache import cache
+from django.db.backends.sqlite3.base import Database as sqlite3
+
+
+class BaseBinProvider(BinProvider):
+    # def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
+    #     Class = super()
+    #     get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
+    #     # return cache.get_or_set(f'bin:abspath:{bin_name}', get_abspath_func)
+    #     return get_abspath_func()
+    
+    # def on_get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None:
+    #     Class = super()
+    #     get_version_func = lambda: Class.on_get_version(bin_name, abspath, **context)
+    #     # return cache.get_or_set(f'bin:version:{bin_name}:{abspath}', get_version_func)
+    #     return get_version_func()
+
+    def register(self, settings, parent_plugin=None):
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.BINPROVIDERS[self.name] = self
+
+
+class BaseBinary(Binary):
+    binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias='binproviders')
+
+    def register(self, settings, parent_plugin=None):
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.BINARIES[self.name] = self
+
+# def get_ytdlp_version() -> str:
+#     import yt_dlp
+#     return yt_dlp.version.__version__
+
+
+
+
+# class YtdlpBinary(Binary):
+#     name: BinName = 'yt-dlp'
+#     providers_supported: List[BinProvider] = [
+#         EnvProvider(),
+#         PipProvider(),
+#         BrewProvider(),
+#         AptProvider(),
+#     ]
+#     provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+#         'pip': {
+#             'version': get_ytdlp_version,
+#         },
+#         'brew': {
+#             'subdeps': lambda: 'yt-dlp ffmpeg',
+#         },
+#         'apt': {
+#             'subdeps': lambda: 'yt-dlp ffmpeg',
+#         }
+#     }
+
+# class WgetBinary(Binary):
+#     name: BinName = 'wget'
+#     providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
+
+
+# if __name__ == '__main__':
+#     PYTHON_BINARY = PythonBinary()
+#     SQLITE_BINARY = SqliteBinary()
+#     DJANGO_BINARY = DjangoBinary()
+#     WGET_BINARY = WgetBinary()
+#     YTDLP_BINARY = YtdlpPBinary()
+
+#     print('-------------------------------------DEFINING BINARIES---------------------------------')
+#     print(PYTHON_BINARY)
+#     print(SQLITE_BINARY)
+#     print(DJANGO_BINARY)
+#     print(WGET_BINARY)
+#     print(YTDLP_BINARY)

archivebox/plugantic/base_check.py

@@ -0,0 +1,55 @@
+from typing import List, Type, Any
+
+from pydantic_core import core_schema
+from pydantic import GetCoreSchemaHandler
+
+from django.utils.functional import classproperty
+from django.core.checks import Warning, Tags, register
+
+class BaseCheck:
+    label: str = ''
+    tag = Tags.database
+    
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
+        return core_schema.typed_dict_schema(
+            {
+                'name': core_schema.typed_dict_field(core_schema.str_schema()),
+                'tag': core_schema.typed_dict_field(core_schema.str_schema()),
+            },
+        )
+
+
+    @classproperty
+    def name(cls) -> str:
+        return cls.label or cls.__name__
+    
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        """Override this method to implement your custom runtime check."""
+        errors = []
+        # if not hasattr(settings, 'SOME_KEY'):
+        #     errors.extend(Error(
+        #         'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?',
+        #         id='core.C001',
+        #         hint='Make sure to run django_setup() is able to load settings.SOME_KEY.',
+        #     ))
+        # logger.debug('[√] Loaded settings.PLUGINS succesfully.')
+        return errors
+
+    def register(self, settings, parent_plugin=None):
+        # Regsiter in ArchiveBox plugins runtime settings
+        self._plugin = parent_plugin
+        settings.CHECKS[self.name] = self
+
+        # Register using Django check framework
+        def run_check(app_configs, **kwargs) -> List[Warning]:
+            from django.conf import settings
+            import logging
+            settings = settings
+            logger = logging.getLogger('checks')
+            return self.check(settings, logger)
+
+        run_check.__name__ = self.label or self.__class__.__name__
+        run_check.tags = [self.tag]
+        register(self.tag)(run_check)

archivebox/plugantic/base_configset.py

@@ -0,0 +1,81 @@
+__package__ = 'archivebox.plugantic'
+
+
+from typing import Optional, List, Literal
+from pathlib import Path
+from pydantic import BaseModel, Field, ConfigDict, computed_field
+
+
+ConfigSectionName = Literal[
+    'GENERAL_CONFIG',
+    'ARCHIVE_METHOD_TOGGLES',
+    'ARCHIVE_METHOD_OPTIONS',
+    'DEPENDENCY_CONFIG',
+]
+ConfigSectionNames: List[ConfigSectionName] = [
+    'GENERAL_CONFIG',
+    'ARCHIVE_METHOD_TOGGLES',
+    'ARCHIVE_METHOD_OPTIONS',
+    'DEPENDENCY_CONFIG',
+]
+
+
+class BaseConfigSet(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True)
+
+    section: ConfigSectionName = 'GENERAL_CONFIG'
+
+    @computed_field
+    @property
+    def name(self) -> str:
+        return self.__class__.__name__
+    
+    def register(self, settings, parent_plugin=None):
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.CONFIGS[self.name] = self
+
+
+
+# class WgetToggleConfig(ConfigSet):
+#     section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
+
+#     SAVE_WGET: bool = True
+#     SAVE_WARC: bool = True
+
+# class WgetDependencyConfig(ConfigSet):
+#     section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+#     WGET_BINARY: str = Field(default='wget')
+#     WGET_ARGS: Optional[List[str]] = Field(default=None)
+#     WGET_EXTRA_ARGS: List[str] = []
+#     WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+# class WgetOptionsConfig(ConfigSet):
+#     section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
+
+#     # loaded from shared config
+#     WGET_AUTO_COMPRESSION: bool = Field(default=True)
+#     SAVE_WGET_REQUISITES: bool = Field(default=True)
+#     WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
+#     WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
+#     WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
+#     WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
+#     WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
+
+
+# CONFIG = {
+#     'CHECK_SSL_VALIDITY': False,
+#     'SAVE_WARC': False,
+#     'TIMEOUT': 999,
+# }
+
+
+# WGET_CONFIG = [
+#     WgetToggleConfig(**CONFIG),
+#     WgetDependencyConfig(**CONFIG),
+#     WgetOptionsConfig(**CONFIG),
+# ]

archivebox/plugantic/extractors.py → archivebox/plugantic/base_extractor.py

@@ -6,13 +6,14 @@ from typing_extensions import Self
 from abc import ABC
 from pathlib import Path
 
-from pydantic import BaseModel, model_validator, field_serializer, AfterValidator
+from pydantic import BaseModel, model_validator, field_serializer, AfterValidator, Field
+from pydantic_pkgr import BinName
 
-from .binaries import (
-    Binary,
-    YtdlpBinary,
-    WgetBinary,
-)
+# from .binaries import (
+#     Binary,
+#     YtdlpBinary,
+#     WgetBinary,
+# )
 
 
 # stubs
@@ -37,9 +38,9 @@ HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
 CmdArgsList = Annotated[List[str], AfterValidator(no_empty_args)]
 
 
-class Extractor(ABC, BaseModel):
+class BaseExtractor(ABC, BaseModel):
     name: ExtractorName
-    binary: Binary
+    binary: BinName
 
     output_path_func: HandlerFuncStr = 'self.get_output_path'
     should_extract_func: HandlerFuncStr = 'self.should_extract'
@@ -55,10 +56,14 @@ class Extractor(ABC, BaseModel):
         if self.args is None:
             self.args = [*self.default_args, *self.extra_args]
         return self
+    
+    def register(self, settings, parent_plugin=None):
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
 
-    @field_serializer('binary', when_used='json')
-    def dump_binary(binary) -> str:
-        return binary.name
+        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.EXTRACTORS[self.name] = self
 
     def get_output_path(self, snapshot) -> Path:
         return Path(self.name)
@@ -86,33 +91,37 @@ class Extractor(ABC, BaseModel):
             'returncode': proc.returncode,
         }
 
-    def exec(self, args: CmdArgsList, pwd: Optional[Path]=None):
+    def exec(self, args: CmdArgsList, pwd: Optional[Path]=None, settings=None):
         pwd = pwd or Path('.')
-        assert self.binary.loaded_provider
-        return self.binary.exec(args, pwd=pwd)
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+        
+        binary = settings.BINARIES[self.binary]
+        return binary.exec(args, pwd=pwd)
 
 
-class YtdlpExtractor(Extractor):
-    name: ExtractorName = 'media'
-    binary: Binary = YtdlpBinary()
+# class YtdlpExtractor(Extractor):
+#     name: ExtractorName = 'media'
+#     binary: Binary = YtdlpBinary()
 
-    def get_output_path(self, snapshot) -> Path:
-        return Path(self.name)
+#     def get_output_path(self, snapshot) -> Path:
+#         return Path(self.name)
 
 
-class WgetExtractor(Extractor):
-    name: ExtractorName = 'wget'
-    binary: Binary = WgetBinary()
+# class WgetExtractor(Extractor):
+#     name: ExtractorName = 'wget'
+#     binary: Binary = WgetBinary()
 
-    def get_output_path(self, snapshot) -> Path:
-        return get_wget_output_path(snapshot)
+#     def get_output_path(self, snapshot) -> Path:
+#         return get_wget_output_path(snapshot)
 
 
-class WarcExtractor(Extractor):
-    name: ExtractorName = 'warc'
-    binary: Binary = WgetBinary()
+# class WarcExtractor(Extractor):
+#     name: ExtractorName = 'warc'
+#     binary: Binary = WgetBinary()
 
-    def get_output_path(self, snapshot) -> Path:
-        return get_wget_output_path(snapshot)
+#     def get_output_path(self, snapshot) -> Path:
+#         return get_wget_output_path(snapshot)
 
 

archivebox/plugantic/base_plugin.py

@@ -0,0 +1,202 @@
+__package__ = 'archivebox.plugantic'
+
+import json
+
+from django.apps import AppConfig
+from django.core.checks import register
+
+from typing import List, ClassVar, Type, Dict
+from typing_extensions import Self
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    model_validator,
+    InstanceOf,
+    computed_field,
+    validate_call,
+)
+
+from .base_configset import BaseConfigSet
+from .base_binary import BaseBinProvider, BaseBinary
+from .base_extractor import BaseExtractor
+from .base_replayer import BaseReplayer
+from .base_check import BaseCheck
+from .base_admindataview import BaseAdminDataView
+
+from ..config import ANSI, AttrDict
+
+
+class BasePlugin(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
+
+    # Required by AppConfig:
+    name: str = Field()                           # e.g. 'builtin_plugins.singlefile'
+    app_label: str = Field()                      # e.g. 'singlefile'
+    verbose_name: str = Field()                   # e.g. 'SingleFile'
+    default_auto_field: ClassVar[str] = 'django.db.models.AutoField'
+    
+    # Required by Plugantic:
+    configs: List[InstanceOf[BaseConfigSet]] = Field(default=[])
+    binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[])                # e.g. [Binary(name='yt-dlp')]
+    binaries: List[InstanceOf[BaseBinary]] = Field(default=[])                # e.g. [Binary(name='yt-dlp')]
+    extractors: List[InstanceOf[BaseExtractor]] = Field(default=[])
+    replayers: List[InstanceOf[BaseReplayer]] = Field(default=[])
+    checks: List[InstanceOf[BaseCheck]] = Field(default=[])
+    admindataviews: List[InstanceOf[BaseAdminDataView]] = Field(default=[])
+
+    @model_validator(mode='after')
+    def validate(self) -> Self:
+        """Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
+        
+        assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
+        
+        assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
+    
+    @property
+    def AppConfig(plugin_self) -> Type[AppConfig]:
+        """Generate a Django AppConfig class for this plugin."""
+
+        class PluginAppConfig(AppConfig):
+            name = plugin_self.name
+            app_label = plugin_self.app_label
+            verbose_name = plugin_self.verbose_name
+        
+            def ready(self):
+                from django.conf import settings
+                
+                plugin_self.validate()
+                plugin_self.register(settings)
+
+        return PluginAppConfig
+    
+    @computed_field
+    @property
+    def BINPROVIDERS(self) -> Dict[str, BaseBinProvider]:
+        return AttrDict({binprovider.name: binprovider for binprovider in self.binproviders})
+    
+    @computed_field
+    @property
+    def BINARIES(self) -> Dict[str, BaseBinary]:
+        return AttrDict({binary.python_name: binary for binary in self.binaries})
+    
+    @computed_field
+    @property
+    def CONFIGS(self) -> Dict[str, BaseConfigSet]:
+        return AttrDict({config.name: config for config in self.configs})
+    
+    @computed_field
+    @property
+    def EXTRACTORS(self) -> Dict[str, BaseExtractor]:
+        return AttrDict({extractor.name: extractor for extractor in self.extractors})
+    
+    @computed_field
+    @property
+    def REPLAYERS(self) -> Dict[str, BaseReplayer]:
+        return AttrDict({replayer.name: replayer for replayer in self.replayers})
+    
+    @computed_field
+    @property
+    def CHECKS(self) -> Dict[str, BaseCheck]:
+        return AttrDict({check.name: check for check in self.checks})
+    
+    @computed_field
+    @property
+    def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]:
+        return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews})
+    
+    @computed_field
+    @property
+    def PLUGIN_KEYS(self) -> List[str]:
+        return 
+
+    def register(self, settings=None):
+        """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
+        
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        assert all(hasattr(settings, key) for key in ['PLUGINS', 'CONFIGS', 'BINARIES', 'EXTRACTORS', 'REPLAYERS', 'ADMINDATAVIEWS']), 'Tried to register plugin in settings but couldnt find required global dicts in settings.'
+
+        assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
+
+        assert self.app_label not in settings.PLUGINS, f'Tried to register plugin {self.name} but it conflicts with existing plugin of the same name ({self.app_label}).'
+
+        ### Mutate django.conf.settings... values in-place to include plugin-provided overrides
+        settings.PLUGINS[self.app_label] = self
+
+        for config in self.CONFIGS.values():
+            config.register(settings, parent_plugin=self)
+        
+        for binprovider in self.BINPROVIDERS.values():
+            binprovider.register(settings, parent_plugin=self)
+        
+        for binary in self.BINARIES.values():
+            binary.register(settings, parent_plugin=self)
+        
+        for extractor in self.EXTRACTORS.values():
+            extractor.register(settings, parent_plugin=self)
+
+        for replayer in self.REPLAYERS.values():
+            replayer.register(settings, parent_plugin=self)
+
+        for check in self.CHECKS.values():
+            check.register(settings, parent_plugin=self)
+
+        for admindataview in self.ADMINDATAVIEWS.values():
+            admindataview.register(settings, parent_plugin=self)
+
+        # TODO: add parsers? custom templates? persona fixtures?
+
+        plugin_prefix, plugin_shortname = self.name.split('.', 1)
+
+        print(
+            f'    > {ANSI.black}{plugin_prefix.upper().replace("_PLUGINS", "").ljust(15)} ' +
+            f'{ANSI.lightyellow}{plugin_shortname.ljust(12)} ' + 
+            f'{ANSI.black}CONFIGSx{len(self.configs)}  BINARIESx{len(self.binaries)}  EXTRACTORSx{len(self.extractors)}  REPLAYERSx{len(self.replayers)}  CHECKSx{len(self.CHECKS)}  ADMINDATAVIEWSx{len(self.ADMINDATAVIEWS)}{ANSI.reset}'
+        )
+
+    # @validate_call
+    # def install_binaries(self) -> Self:
+    #     new_binaries = []
+    #     for idx, binary in enumerate(self.binaries):
+    #         new_binaries.append(binary.install() or binary)
+    #     return self.model_copy(update={
+    #         'binaries': new_binaries,
+    #     })
+
+    @validate_call
+    def load_binaries(self, cache=True) -> Self:
+        new_binaries = []
+        for idx, binary in enumerate(self.binaries):
+            new_binaries.append(binary.load(cache=cache) or binary)
+        return self.model_copy(update={
+            'binaries': new_binaries,
+        })
+
+    # @validate_call
+    # def load_or_install_binaries(self, cache=True) -> Self:
+    #     new_binaries = []
+    #     for idx, binary in enumerate(self.binaries):
+    #         new_binaries.append(binary.load_or_install(cache=cache) or binary)
+    #     return self.model_copy(update={
+    #         'binaries': new_binaries,
+    #     })
+
+
+
+
+# class YtdlpPlugin(BasePlugin):
+#     name: str = 'ytdlp'
+#     configs: List[SerializeAsAny[BaseConfigSet]] = []
+#     binaries: List[SerializeAsAny[BaseBinary]] = [YtdlpBinary()]
+#     extractors: List[SerializeAsAny[BaseExtractor]] = [YtdlpExtractor()]
+#     replayers: List[SerializeAsAny[BaseReplayer]] = [MEDIA_REPLAYER]
+
+# class WgetPlugin(BasePlugin):
+#     name: str = 'wget'
+#     configs: List[SerializeAsAny[BaseConfigSet]] = [*WGET_CONFIG]
+#     binaries: List[SerializeAsAny[BaseBinary]] = [WgetBinary()]
+#     extractors: List[SerializeAsAny[BaseExtractor]] = [WgetExtractor(), WarcExtractor()]

archivebox/plugantic/replayers.py → archivebox/plugantic/base_replayer.py

@@ -3,10 +3,9 @@ __package__ = 'archivebox.plugantic'
 
 from pydantic import BaseModel
 
-# from .binproviders import LazyImportStr
 
 
-class Replayer(BaseModel):
+class BaseReplayer(BaseModel):
     """Describes how to render an ArchiveResult in several contexts"""
     name: str = 'GenericReplayer'
     url_pattern: str = '*'
@@ -21,5 +20,17 @@ class Replayer(BaseModel):
     # icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
     # thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
 
+    def register(self, settings, parent_plugin=None):
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
 
-MEDIA_REPLAYER = Replayer(name='media')
+        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.REPLAYERS[self.name] = self
+
+
+# class MediaReplayer(BaseReplayer):
+#     name: str = 'MediaReplayer'
+
+
+# MEDIA_REPLAYER = MediaReplayer()

archivebox/plugantic/binaries.py

@@ -1,65 +0,0 @@
-__package__ = 'archivebox.plugantic'
-
-import sys
-import inspect
-import importlib
-from pathlib import Path
-
-
-from typing import Any, Optional, Dict, List
-from typing_extensions import Self
-from subprocess import run, PIPE
-
-from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
-
-import django
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-
-
-
-def get_ytdlp_version() -> str:
-    import yt_dlp
-    return yt_dlp.version.__version__
-
-
-
-
-class YtdlpBinary(Binary):
-    name: BinName = 'yt-dlp'
-    providers_supported: List[BinProvider] = [
-        EnvProvider(),
-        PipProvider(),
-        BrewProvider(),
-        AptProvider(),
-    ]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'pip': {
-            'version': get_ytdlp_version,
-        },
-        'brew': {
-            'subdeps': lambda: 'yt-dlp ffmpeg',
-        },
-        'apt': {
-            'subdeps': lambda: 'yt-dlp ffmpeg',
-        }
-    }
-
-class WgetBinary(Binary):
-    name: BinName = 'wget'
-    providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
-
-
-# if __name__ == '__main__':
-#     PYTHON_BINARY = PythonBinary()
-#     SQLITE_BINARY = SqliteBinary()
-#     DJANGO_BINARY = DjangoBinary()
-#     WGET_BINARY = WgetBinary()
-#     YTDLP_BINARY = YtdlpPBinary()
-
-#     print('-------------------------------------DEFINING BINARIES---------------------------------')
-#     print(PYTHON_BINARY)
-#     print(SQLITE_BINARY)
-#     print(DJANGO_BINARY)
-#     print(WGET_BINARY)
-#     print(YTDLP_BINARY)

archivebox/plugantic/configs.py

@@ -1,53 +0,0 @@
-__package__ = 'archivebox.plugantic'
-
-
-from typing import Optional, List, Literal
-from pathlib import Path
-from pydantic import BaseModel, Field
-
-
-ConfigSectionName = Literal['GENERAL_CONFIG', 'ARCHIVE_METHOD_TOGGLES', 'ARCHIVE_METHOD_OPTIONS', 'DEPENDENCY_CONFIG']
-
-
-class ConfigSet(BaseModel):
-    section: ConfigSectionName = 'GENERAL_CONFIG'
-
-class WgetToggleConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
-
-    SAVE_WGET: bool = True
-    SAVE_WARC: bool = True
-
-class WgetDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    WGET_BINARY: str = Field(default='wget')
-    WGET_ARGS: Optional[List[str]] = Field(default=None)
-    WGET_EXTRA_ARGS: List[str] = []
-    WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class WgetOptionsConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
-
-    # loaded from shared config
-    WGET_AUTO_COMPRESSION: bool = Field(default=True)
-    SAVE_WGET_REQUISITES: bool = Field(default=True)
-    WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
-    WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
-    WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
-    WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
-    WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
-
-
-CONFIG = {
-    'CHECK_SSL_VALIDITY': False,
-    'SAVE_WARC': False,
-    'TIMEOUT': 999,
-}
-
-
-WGET_CONFIG = [
-    WgetToggleConfig(**CONFIG),
-    WgetDependencyConfig(**CONFIG),
-    WgetOptionsConfig(**CONFIG),
-]

archivebox/plugantic/migrations/0001_initial.py

@@ -1,38 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 00:16
-
-import abid_utils.models
-import archivebox.plugantic.plugins
-import charidfield.fields
-import django.core.serializers.json
-import django.db.models.deletion
-import django_pydantic_field.fields
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Plugin',
-            fields=[
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('uuid', models.UUIDField(blank=True, null=True, unique=True)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
-                ('schema', django_pydantic_field.fields.PydanticSchemaField(config=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin)),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'abstract': False,
-            },
-        ),
-    ]

archivebox/plugantic/migrations/0002_alter_plugin_schema.py

@@ -1,21 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:16
-
-import archivebox.plugantic.plugins
-import django.core.serializers.json
-import django_pydantic_field.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='plugin',
-            name='schema',
-            field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin),
-        ),
-    ]

archivebox/plugantic/migrations/0003_alter_plugin_schema.py

@@ -1,21 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:25
-
-import archivebox.plugantic.replayers
-import django.core.serializers.json
-import django_pydantic_field.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0002_alter_plugin_schema'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='plugin',
-            name='schema',
-            field=django_pydantic_field.fields.PydanticSchemaField(config=None, default={'embed_template': 'plugins/generic_replayer/templates/embed.html', 'fullpage_template': 'plugins/generic_replayer/templates/fullpage.html', 'name': 'GenericReplayer', 'row_template': 'plugins/generic_replayer/templates/row.html', 'url_pattern': '*'}, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.replayers.Replayer),
-        ),
-    ]

archivebox/plugantic/migrations/0004_remove_plugin_schema_plugin_configs_plugin_name.py

@@ -1,32 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:28
-
-import archivebox.plugantic.configs
-import django.core.serializers.json
-import django_pydantic_field.compat.django
-import django_pydantic_field.fields
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0003_alter_plugin_schema'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='plugin',
-            name='schema',
-        ),
-        migrations.AddField(
-            model_name='plugin',
-            name='configs',
-            field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=[], encoder=django.core.serializers.json.DjangoJSONEncoder, schema=django_pydantic_field.compat.django.GenericContainer(list, (archivebox.plugantic.configs.ConfigSet,))),
-        ),
-        migrations.AddField(
-            model_name='plugin',
-            name='name',
-            field=models.CharField(default='name', max_length=64, unique=True),
-            preserve_default=False,
-        ),
-    ]

archivebox/plugantic/migrations/0005_customplugin_delete_plugin.py

@@ -1,39 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:42
-
-import abid_utils.models
-import charidfield.fields
-import django.db.models.deletion
-import pathlib
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0004_remove_plugin_schema_plugin_configs_plugin_name'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='CustomPlugin',
-            fields=[
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('uuid', models.UUIDField(blank=True, null=True, unique=True)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
-                ('name', models.CharField(max_length=64, unique=True)),
-                ('path', models.FilePathField(path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'))),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'abstract': False,
-            },
-        ),
-        migrations.DeleteModel(
-            name='Plugin',
-        ),
-    ]

archivebox/plugantic/migrations/0006_alter_customplugin_path.py

@@ -1,19 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:45
-
-import pathlib
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0005_customplugin_delete_plugin'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'), recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0007_alter_customplugin_path.py

@@ -1,19 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:46
-
-import pathlib
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0006_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins'), recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0008_alter_customplugin_path.py

@@ -1,19 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:47
-
-import pathlib
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0007_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data'), recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0009_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:48
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0008_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0010_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:48
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0009_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, match='/plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0011_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:48
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0010_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0012_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:49
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0011_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, default='example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0013_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:49
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0012_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0014_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:50
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0013_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0015_alter_customplugin_path.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:51
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0014_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customplugin',
-            name='path',
-            field=models.FilePathField(allow_files=False, allow_folders=True, match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True),
-        ),
-    ]

archivebox/plugantic/migrations/0016_delete_customplugin.py

@@ -1,16 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-18 01:57
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('plugantic', '0015_alter_customplugin_path'),
-    ]
-
-    operations = [
-        migrations.DeleteModel(
-            name='CustomPlugin',
-        ),
-    ]

archivebox/plugantic/plugins.py

@@ -1,122 +0,0 @@
-__package__ = 'archivebox.plugantic'
-
-from typing import List
-from typing_extensions import Self
-
-from pydantic import (
-    BaseModel,
-    ConfigDict,
-    Field,
-    model_validator,
-    validate_call,
-    SerializeAsAny,
-)
-
-from .binaries import (
-    Binary,
-    WgetBinary,
-    YtdlpBinary,
-)
-from .extractors import (
-    Extractor,
-    YtdlpExtractor,
-    WgetExtractor,
-    WarcExtractor,
-)
-from .replayers import (
-    Replayer,
-    MEDIA_REPLAYER,
-)
-from .configs import (
-    ConfigSet,
-    WGET_CONFIG,
-)
-
-
-class Plugin(BaseModel):
-    model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
-
-    name: str = Field(default='baseplugin')                   # e.g. media
-    description: str = Field(default='')                      # e.g. get media using yt-dlp
-    
-    configs: List[SerializeAsAny[ConfigSet]] = Field(default=[])
-    binaries: List[SerializeAsAny[Binary]] = Field(default=[])                # e.g. [Binary(name='yt-dlp')]
-    extractors: List[SerializeAsAny[Extractor]] = Field(default=[])
-    replayers: List[SerializeAsAny[Replayer]] = Field(default=[])
-
-    @model_validator(mode='after')
-    def validate(self):
-        self.description = self.description or self.name
-
-    @validate_call
-    def install(self) -> Self:
-        new_binaries = []
-        for idx, binary in enumerate(self.binaries):
-            new_binaries.append(binary.install() or binary)
-        return self.model_copy(update={
-            'binaries': new_binaries,
-        })
-
-    @validate_call
-    def load(self, cache=True) -> Self:
-        new_binaries = []
-        for idx, binary in enumerate(self.binaries):
-            new_binaries.append(binary.load(cache=cache) or binary)
-        return self.model_copy(update={
-            'binaries': new_binaries,
-        })
-
-    @validate_call
-    def load_or_install(self, cache=True) -> Self:
-        new_binaries = []
-        for idx, binary in enumerate(self.binaries):
-            new_binaries.append(binary.load_or_install(cache=cache) or binary)
-        return self.model_copy(update={
-            'binaries': new_binaries,
-        })
-
-
-
-class YtdlpPlugin(Plugin):
-    name: str = 'ytdlp'
-    configs: List[SerializeAsAny[ConfigSet]] = []
-    binaries: List[SerializeAsAny[Binary]] = [YtdlpBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [YtdlpExtractor()]
-    replayers: List[SerializeAsAny[Replayer]] = [MEDIA_REPLAYER]
-
-class WgetPlugin(Plugin):
-    name: str = 'wget'
-    configs: List[SerializeAsAny[ConfigSet]] = [*WGET_CONFIG]
-    binaries: List[SerializeAsAny[Binary]] = [WgetBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [WgetExtractor(), WarcExtractor()]
-
-
-YTDLP_PLUGIN = YtdlpPlugin()
-WGET_PLUGIN = WgetPlugin()
-PLUGINS = [
-    YTDLP_PLUGIN,
-    WGET_PLUGIN,
-]
-LOADED_PLUGINS = PLUGINS
-
-
-import json
-
-for plugin in PLUGINS:
-    try:
-        json.dumps(plugin.model_json_schema(), indent=4)
-        # print(json.dumps(plugin.model_json_schema(), indent=4))
-    except Exception as err:
-        print(f'Failed to generate JSON schema for {plugin.name}')
-        raise
-
-# print('-------------------------------------BEFORE INSTALL---------------------------------')
-# for plugin in PLUGINS:
-#     print(plugin.model_dump_json(indent=4))
-# print('-------------------------------------DURING LOAD/INSTALL---------------------------------')
-# for plugin in PLUGINS:
-    # LOADED_PLUGINS.append(plugin.install())
-# print('-------------------------------------AFTER INSTALL---------------------------------')
-# for plugin in LOADED_PLUGINS:
-    # print(plugin.model_dump_json(indent=4))
-

archivebox/plugantic/views.py

@@ -4,17 +4,19 @@ import inspect
 from typing import Any
 
 from django.http import HttpRequest
+from django.conf import settings
 from django.utils.html import format_html, mark_safe
 
 from admin_data_views.typing import TableContext, ItemContext
 from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
 
 
-from plugantic.plugins import LOADED_PLUGINS
 from django.conf import settings
 
 def obj_to_yaml(obj: Any, indent: int=0) -> str:
     indent_str = "  " * indent
+    if indent == 0:
+        indent_str = '\n'  # put extra newline between top-level entries
     
     if isinstance(obj, dict):
         if not obj:
@@ -74,22 +76,34 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
         if '_BINARY' in key or '_VERSION' in key
     }
 
-    for plugin in LOADED_PLUGINS:
+    for plugin in settings.PLUGINS.values():
         for binary in plugin.binaries:
-            binary = binary.load_or_install()
+            try:
+                binary = binary.load()
+            except Exception as e:
+                print(e)
 
             rows['Binary'].append(ItemLink(binary.name, key=binary.name))
-            rows['Found Version'].append(binary.loaded_version)
+            rows['Found Version'].append(f'✅ {binary.loaded_version}' if binary.loaded_version else '❌ missing')
             rows['From Plugin'].append(plugin.name)
-            rows['Provided By'].append(binary.loaded_provider)
-            rows['Found Abspath'].append(binary.loaded_abspath)
+            rows['Provided By'].append(
+                ', '.join(
+                    f'[{binprovider.name}]' if binprovider.name == getattr(binary.loaded_binprovider, 'name', None) else binprovider.name
+                    for binprovider in binary.binproviders_supported
+                    if binprovider
+                )
+                # binary.loaded_binprovider.name
+                # if binary.loaded_binprovider else
+                # ', '.join(getattr(provider, 'name', str(provider)) for provider in binary.binproviders_supported)
+            )
+            rows['Found Abspath'].append(binary.loaded_abspath or '❌ missing')
             rows['Related Configuration'].append(mark_safe(', '.join(
                 f'<a href="/admin/environment/config/{config_key}/">{config_key}</a>'
                 for config_key, config_value in relevant_configs.items()
                     if binary.name.lower().replace('-', '').replace('_', '').replace('ytdlp', 'youtubedl') in config_key.lower()
                     # or binary.name.lower().replace('-', '').replace('_', '') in str(config_value).lower()
             )))
-            rows['Overrides'].append(obj_to_yaml(binary.provider_overrides))
+            rows['Overrides'].append(str(obj_to_yaml(binary.provider_overrides))[:200])
             # rows['Description'].append(binary.description)
 
     return TableContext(
@@ -104,7 +118,7 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
     binary = None
     plugin = None
-    for loaded_plugin in LOADED_PLUGINS:
+    for loaded_plugin in settings.PLUGINS.values():
         for loaded_binary in loaded_plugin.binaries:
             if loaded_binary.name == key:
                 binary = loaded_binary
@@ -112,7 +126,10 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
     assert plugin and binary, f'Could not find a binary matching the specified name: {key}'
 
-    binary = binary.load_or_install()
+    try:
+        binary = binary.load()
+    except Exception as e:
+        print(e)
 
     return ItemContext(
         slug=key,
@@ -120,14 +137,14 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         data=[
             {
                 "name": binary.name,
-                "description": binary.description,
+                "description": binary.abspath,
                 "fields": {
                     'plugin': plugin.name,
-                    'binprovider': binary.loaded_provider,
+                    'binprovider': binary.loaded_binprovider,
                     'abspath': binary.loaded_abspath,
                     'version': binary.loaded_version,
                     'overrides': obj_to_yaml(binary.provider_overrides),
-                    'providers': obj_to_yaml(binary.providers_supported),
+                    'providers': obj_to_yaml(binary.binproviders_supported),
                 },
                 "help_texts": {
                     # TODO
@@ -148,12 +165,15 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
         "extractors": [],
         "replayers": [],
         "configs": [],
-        "description": [],
+        "verbose_name": [],
     }
 
 
-    for plugin in LOADED_PLUGINS:
-        plugin = plugin.load_or_install()
+    for plugin in settings.PLUGINS.values():
+        try:
+            plugin = plugin.load_binaries()
+        except Exception as e:
+            print(e)
 
         rows['Name'].append(ItemLink(plugin.name, key=plugin.name))
         rows['binaries'].append(mark_safe(', '.join(
@@ -168,7 +188,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 for config_key in configset.__fields__.keys()
                     if config_key != 'section' and config_key in settings.CONFIG
         )))
-        rows['description'].append(str(plugin.description))
+        rows['verbose_name'].append(str(plugin.verbose_name))
 
     return TableContext(
         title="Installed plugins",
@@ -181,13 +201,16 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
 
     plugin = None
-    for loaded_plugin in LOADED_PLUGINS:
+    for loaded_plugin in settings.PLUGINS.values():
         if loaded_plugin.name == key:
             plugin = loaded_plugin
 
     assert plugin, f'Could not find a plugin matching the specified name: {key}'
 
-    plugin = plugin.load_or_install()
+    try:
+        plugin = plugin.load_binaries()
+    except Exception as e:
+        print(e)
 
     return ItemContext(
         slug=key,
@@ -195,12 +218,13 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         data=[
             {
                 "name": plugin.name,
-                "description": plugin.description,
+                "description": plugin.verbose_name,
                 "fields": {
                     'configs': plugin.configs,
                     'binaries': plugin.binaries,
                     'extractors': plugin.extractors,
                     'replayers': plugin.replayers,
+                    'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', settings.PLUGIN_KEYS.keys()))),
                 },
                 "help_texts": {
                     # TODO

archivebox/vendor/pydantic-pkgr

@@ -1 +1 @@
-Subproject commit c97de57f8df5f36a0f8cd1e51645f114e74bffb0
+Subproject commit 36aaa4f9098e5987e23394398aa56154582bd2d2