fix config loading precedence order

archivebox/builtin_plugins/npm/apps.py

@@ -40,11 +40,18 @@ npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
 
 class NpmBinary(BaseBinary):
     name: BinName = 'npm'
-    binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
 
 
 NPM_BINARY = NpmBinary()
 
+class NodeBinary(BaseBinary):
+    name: BinName = 'node'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
+
+
+NODE_BINARY = NodeBinary()
+
 
 
 class NpmPlugin(BasePlugin):
@@ -54,7 +61,7 @@ class NpmPlugin(BasePlugin):
 
     configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG]
     binproviders: List[InstanceOf[BaseBinProvider]] = [npm]
-    binaries: List[InstanceOf[BaseBinary]] = [NPM_BINARY]
+    binaries: List[InstanceOf[BaseBinary]] = [NODE_BINARY, NPM_BINARY]
 
 
 PLUGIN = NpmPlugin()

archivebox/builtin_plugins/pip/apps.py

@@ -1,13 +1,19 @@
 import sys
+import inspect
 from pathlib import Path
 from typing import List, Dict, Optional
 from pydantic import InstanceOf, Field
 
+import django
 from django.apps import AppConfig
 
-from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr
+from django.db.backends.sqlite3.base import Database as sqlite3
+from django.core.checks import Error, Tags, register
+
+from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
 from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
 from plugantic.base_configset import ConfigSectionName
+from plugantic.base_check import BaseCheck
 
 from pkg.settings import env, apt, brew
 
@@ -37,13 +43,75 @@ pip = PipProvider(PATH=str(Path(sys.executable).parent))
 
 class PipBinary(BaseBinary):
     name: BinName = 'pip'
-    binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
 PIP_BINARY = PipBinary()
 
 
 
 
 
+class PythonBinary(BaseBinary):
+    name: BinName = 'python'
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
+        'apt': {
+            'subdeps': \
+                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+class SqliteBinary(BaseBinary):
+    name: BinName = 'sqlite'
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: Path(inspect.getfile(sqlite3)),
+            'version': \
+                lambda: SemVer(sqlite3.version),
+        },
+    }
+
+
+class DjangoBinary(BaseBinary):
+    name: BinName = 'django'
+
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+
+
+
+class CheckUserIsNotRoot(BaseCheck):
+    label: str = 'CheckUserIsNotRoot'
+    tag = Tags.database
+
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        errors = []
+        if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0:
+            errors.append(
+                Error(
+                    "Cannot run as root!",
+                    id="core.S001",
+                    hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).',
+                )
+            )
+        logger.debug('[√] UID is not root')
+        return errors
+
 
 
 
@@ -54,7 +122,8 @@ class PipPlugin(BasePlugin):
 
     configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG]
     binproviders: List[InstanceOf[BaseBinProvider]] = [pip]
-    binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY]
+    binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY, PythonBinary(), SqliteBinary(), DjangoBinary()]
+    checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()]
 
 
 PLUGIN = PipPlugin()

archivebox/builtin_plugins/singlefile/apps.py

@@ -71,7 +71,7 @@ def get_singlefile_abspath() -> Optional[Path]:
 
 class SinglefileBinary(BaseBinary):
     name: BinName = 'single-file'
-    binproviders_supported: List[InstanceOf[BinProvider]] = [env, npm]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [npm, env]
 
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={
         # 'env': {

archivebox/builtin_plugins/systempython/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

archivebox/builtin_plugins/systempython/apps.py

@@ -1,116 +0,0 @@
-__package__ = 'archivebox.builtin_plugins.systempython'
-
-import os
-import sys
-import inspect
-from typing import List, Dict, Any, Callable, ClassVar
-from pathlib import Path
-
-import django
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-from django.utils.functional import classproperty
-from django.db.backends.sqlite3.base import Database as sqlite3
-from django.core.checks import Tags, Error, register
-
-from pydantic import InstanceOf, Field
-
-from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider, BaseExtractor, BaseReplayer
-from plugantic.base_check import BaseCheck
-
-from pkg.settings import env, apt, brew
-
-from builtin_plugins.pip.apps import pip
-
-class PythonBinary(BaseBinary):
-    name: BinName = 'python'
-
-    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
-    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
-        'apt': {
-            'subdeps': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
-            'abspath': \
-                lambda: sys.executable,
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
-        },
-    }
-
-class SqliteBinary(BaseBinary):
-    name: BinName = 'sqlite'
-    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'pip': {
-            'abspath': \
-                lambda: Path(inspect.getfile(sqlite3)),
-            'version': \
-                lambda: SemVer(sqlite3.version),
-        },
-    }
-
-
-class DjangoBinary(BaseBinary):
-    name: BinName = 'django'
-
-    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'pip': {
-            'abspath': \
-                lambda: inspect.getfile(django),
-            'version': \
-                lambda: django.VERSION[:3],
-        },
-    }
-
-
-class BasicReplayer(BaseReplayer):
-    name: str = 'basic'
-
-
-
-
-class CheckUserIsNotRoot(BaseCheck):
-    label: str = 'CheckUserIsNotRoot'
-    tag = Tags.database
-
-    @staticmethod
-    def check(settings, logger) -> List[Warning]:
-        errors = []
-        if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0:
-            errors.append(
-                Error(
-                    "Cannot run as root!",
-                    id="core.S001",
-                    hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).',
-                )
-            )
-        logger.debug('[√] UID is not root')
-        return errors
-
-
-
-class SystemPythonPlugin(BasePlugin):
-    name: str = 'builtin_plugins.systempython'
-    app_label: str = 'systempython'
-    verbose_name: str = 'System Python'
-
-    configs: List[InstanceOf[BaseConfigSet]] = []
-    binaries: List[InstanceOf[BaseBinary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
-    extractors: List[InstanceOf[BaseExtractor]] = []
-    replayers: List[InstanceOf[BaseReplayer]] = [BasicReplayer()]
-    checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()]
-
-
-PLUGIN = SystemPythonPlugin()
-DJANGO_APP = PLUGIN.AppConfig
-# CONFIGS = PLUGIN.configs
-# BINARIES = PLUGIN.binaries
-# EXTRACTORS = PLUGIN.extractors
-# REPLAYERS = PLUGIN.replayers
-# PARSERS = PLUGIN.parsers
-# DAEMONS = PLUGIN.daemons
-# MODELS = PLUGIN.models
-# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/systempython/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

archivebox/builtin_plugins/systempython/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/systempython/views.py

@@ -1,3 +0,0 @@
-from django.shortcuts import render
-
-# Create your views here.

archivebox/builtin_plugins/ytdlp/apps.py

@@ -1,4 +1,5 @@
 import sys
+import shutil
 from pathlib import Path
 from typing import List, Dict, Optional
 from subprocess import run, PIPE, CompletedProcess
@@ -31,21 +32,24 @@ YTDLP_CONFIG = YtdlpDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
 
 class YtdlpBinary(BaseBinary):
     name: BinName = YTDLP_CONFIG.YTDLP_BINARY
-    binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
 
 class FfmpegBinary(BaseBinary):
     name: BinName = 'ffmpeg'
-    binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
 
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
         'env': {
-            'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
+            # 'abspath': lambda: shutil.which('ffmpeg', PATH=env.PATH),
+            # 'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
         },
         'apt': {
-            'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
+            # 'abspath': lambda: shutil.which('ffmpeg', PATH=apt.PATH),
+            'version': lambda: run(['apt', 'show', 'ffmpeg'], stdout=PIPE, stderr=PIPE, text=True).stdout,
         },
         'brew': {
-            'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
+            # 'abspath': lambda: shutil.which('ffmpeg', PATH=brew.PATH),
+            'version': lambda: run(['brew', 'info', 'ffmpeg', '--quiet'], stdout=PIPE, stderr=PIPE, text=True).stdout,
         },
     }
 

archivebox/vendor/pydantic-pkgr

@@ -1 +1 @@
-Subproject commit 36aaa4f9098e5987e23394398aa56154582bd2d2
+Subproject commit ce9c33192df319f843655c80018c4126b5d3fad1