fix LDAP_LIB loading from venv and other site packages dirs

archivebox/plugins_auth/ldap/apps.py

@@ -1,5 +1,6 @@
 __package__ = 'archivebox.plugins_auth.ldap'
 
+
 import inspect
 
 from typing import List, Dict
@@ -12,14 +13,27 @@ from abx.archivebox.base_plugin import BasePlugin
 from abx.archivebox.base_hook import BaseHook
 from abx.archivebox.base_binary import BaseBinary, BaseBinProvider, apt
 
-from plugins_pkg.pip.apps import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER
+from plugins_pkg.pip.apps import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, VENV_SITE_PACKAGES, LIB_SITE_PACKAGES, USER_SITE_PACKAGES, SYS_SITE_PACKAGES
 from .settings import LDAP_CONFIG, get_ldap_lib
 
 
 ###################### Config ##########################
 
-LDAP_LIB = lambda: get_ldap_lib()[0]   # lazy load to avoid slow ldap lib import on startup
-
+def get_LDAP_LIB_path(paths):
+    LDAP_LIB = get_ldap_lib()[0]
+    if not LDAP_LIB:
+        return None
+    
+    # check that LDAP_LIB path is in one of the specified site packages dirs
+    lib_path = Path(inspect.getfile(LDAP_LIB))
+    for site_packges_dir in paths:
+        if str(lib_path.parent.parent.resolve()) == str(Path(site_packges_dir).resolve()):
+            return lib_path
+    return None
+
+def get_LDAP_LIB_version():
+    LDAP_LIB = get_ldap_lib()[0]
+    return LDAP_LIB and SemVer(LDAP_LIB.__version__)
 
 class LdapBinary(BaseBinary):
     name: str = 'ldap'
@@ -27,24 +41,24 @@ class LdapBinary(BaseBinary):
     binproviders_supported: List[InstanceOf[BaseBinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, apt]
 
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
-        VENV_PIP_BINPROVIDER.name: {
-            "abspath": lambda: LDAP_LIB() and Path(inspect.getfile(LDAP_LIB())),         # type: ignore
-            "version": lambda: LDAP_LIB() and SemVer(LDAP_LIB().__version__),            # type: ignore
+        LIB_PIP_BINPROVIDER.name: {
+            "abspath": lambda: get_LDAP_LIB_path(LIB_SITE_PACKAGES),
+            "version": lambda: get_LDAP_LIB_version(),
             "packages": lambda: ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
         },
-        SYS_PIP_BINPROVIDER.name: {
-            "abspath": lambda: LDAP_LIB() and Path(inspect.getfile(LDAP_LIB())),         # type: ignore
-            "version": lambda: LDAP_LIB() and SemVer(LDAP_LIB().__version__),            # type: ignore
+        VENV_PIP_BINPROVIDER.name: {
+            "abspath": lambda: get_LDAP_LIB_path(VENV_SITE_PACKAGES),
+            "version": lambda: get_LDAP_LIB_version(),
             "packages": lambda: ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
         },
-        LIB_PIP_BINPROVIDER.name: {
-            "abspath": lambda: LDAP_LIB() and Path(inspect.getfile(LDAP_LIB())),         # type: ignore
-            "version": lambda: LDAP_LIB() and SemVer(LDAP_LIB().__version__),            # type: ignore
+        SYS_PIP_BINPROVIDER.name: {
+            "abspath": lambda: get_LDAP_LIB_path((*USER_SITE_PACKAGES, *SYS_SITE_PACKAGES)),
+            "version": lambda: get_LDAP_LIB_version(),
             "packages": lambda: ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
         },
         apt.name: {
-            "abspath": lambda: LDAP_LIB() and Path(inspect.getfile(LDAP_LIB())),         # type: ignore
-            "version": lambda: LDAP_LIB() and SemVer(LDAP_LIB().__version__),            # type: ignore
+            "abspath": lambda: get_LDAP_LIB_path(SYS_SITE_PACKAGES),
+            "version": lambda: get_LDAP_LIB_version(),
             "packages": lambda: ['libssl-dev', 'libldap2-dev', 'libsasl2-dev', 'python3-ldap', 'python3-msgpack', 'python3-mutagen'],
         },
     }

archivebox/plugins_auth/ldap/settings.py

@@ -2,8 +2,6 @@ __package__ = 'archivebox.plugins_auth.ldap'
 
 import sys
 
-from functools import cache
-
 from typing import Dict, List, Optional
 from pydantic import Field, model_validator, computed_field
 
@@ -12,12 +10,15 @@ from abx.archivebox.base_configset import BaseConfigSet
 LDAP_LIB = None
 LDAP_SEARCH = None
 
-@cache
-def get_ldap_lib():    
+def get_ldap_lib(extra_paths=()):
     global LDAP_LIB, LDAP_SEARCH
     if LDAP_LIB and LDAP_SEARCH:
         return LDAP_LIB, LDAP_SEARCH
     try:
+        for path in extra_paths:
+            if path not in sys.path:
+                sys.path.append(path)
+            
         import ldap
         from django_auth_ldap.config import LDAPSearch
         LDAP_LIB, LDAP_SEARCH = ldap, LDAPSearch

archivebox/plugins_pkg/pip/apps.py

@@ -2,6 +2,7 @@ __package__ = 'archivebox.plugins_pkg.pip'
 
 import os
 import sys
+import site
 from pathlib import Path
 from typing import List, Dict, Optional
 from pydantic import InstanceOf, Field, model_validator, validate_call
@@ -83,10 +84,23 @@ pip = LIB_PIP_BINPROVIDER
 assert VENV_PIP_BINPROVIDER.pip_venv is not None
 assert LIB_PIP_BINPROVIDER.pip_venv is not None
 
-site_packages_dir = 'lib/python{}.{}/site-packages'.format(*sys.version_info[:2])
-if os.environ.get("VIRTUAL_ENV", None):
-    sys.path.append(str(VENV_PIP_BINPROVIDER.pip_venv / site_packages_dir))
-sys.path.append(str(LIB_PIP_BINPROVIDER.pip_venv / site_packages_dir))
+major, minor, patch = sys.version_info[:3]
+site_packages_dir = f'lib/python{major}.{minor}/site-packages'
+
+LIB_SITE_PACKAGES = (LIB_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
+VENV_SITE_PACKAGES = (VENV_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
+USER_SITE_PACKAGES = site.getusersitepackages()
+SYS_SITE_PACKAGES = site.getsitepackages()
+
+ALL_SITE_PACKAGES = (
+    *LIB_SITE_PACKAGES,
+    *VENV_SITE_PACKAGES,
+    *USER_SITE_PACKAGES,
+    *SYS_SITE_PACKAGES,
+)
+for site_packages_dir in ALL_SITE_PACKAGES:
+    if site_packages_dir not in sys.path:
+        sys.path.append(str(site_packages_dir))
 
 
 class ArchiveboxBinary(BaseBinary):
@@ -94,10 +108,10 @@ class ArchiveboxBinary(BaseBinary):
 
     binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
-        VENV_PIP_BINPROVIDER.name:  {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')},
-        SYS_PIP_BINPROVIDER.name:   {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')},
-        apt.name:                   {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')},
-        brew.name:                  {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')},
+        VENV_PIP_BINPROVIDER.name:  {'packages': lambda: [], 'version': lambda: VERSION},
+        SYS_PIP_BINPROVIDER.name:   {'packages': lambda: [], 'version': lambda: VERSION},
+        apt.name:                   {'packages': lambda: [], 'version': lambda: VERSION},
+        brew.name:                  {'packages': lambda: [], 'version': lambda: VERSION},
     }
     
     @validate_call