Merge pull request #274 from pirate/v0.4.3

v0.4.3 (pre-release, bugfixes for v0.4.0)

MANIFEST.in

@@ -1,8 +1,4 @@
+include LICENSE
+include README.md
 include archivebox/VERSION
-graft archivebox/themes
-graft archivebox/themes/static
-graft archivebox/themes/admin
-graft archivebox/themes/default
-graft archivebox/themes/default/static
-graft archivebox/themes/legacy
-graft archivebox/themes/legacy/static
+recursive-include archivebox/themes *

Pipfile

@@ -12,6 +12,7 @@ setuptools = "*"
 sphinx = "*"
 recommonmark = "*"
 sphinx-rtd-theme = "*"
+twine = "*"
 
 [packages]
 dataclasses = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "8ac4f9e5cd266406a861a283b321b9eee0ca469638f838e93467403ef2f0594d"
+            "sha256": "5a1618caef76ff53b66c5e8674d8e639d25f75068f7026ad799e217d307628fc"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -64,11 +64,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:7c3543e4fb070d14e10926189a7fcf42ba919263b7473dceaefce34d54e8a119",
-                "sha256:a2814bffd1f007805b19194eb0b9a331933b82bd5da1c3ba3d7b7ba16e06dc4b"
+                "sha256:6fcc3cbd55b16f9a01f37de8bcbe286e0ea22e87096557f1511051780338eaea",
+                "sha256:bb407d0bb46395ca1241f829f5bd03f7e482f97f7d1936e26e98dacb201ed4ec"
             ],
             "index": "pypi",
-            "version": "==2.2"
+            "version": "==2.2.1"
         },
         "django-extensions": {
             "hashes": [
@@ -203,11 +203,11 @@
         },
         "youtube-dl": {
             "hashes": [
-                "sha256:46f6e30c673ba71de84748dad4c264d1b6fb30beebf1ef834846a651b4524a78",
-                "sha256:b20d110e1bed8d16f5771bb938ab6e5da67f08af62b599af65301cca290f2e15"
+                "sha256:31844229a4f4d7003e03ab309ff2caff1b16ce0acbd3cfb7a13276058af13056",
+                "sha256:a751bd293e2d7ee963910de14b3eb95b88837021899be488fade0b8abe815650"
             ],
             "index": "pypi",
-            "version": "==2019.4.24"
+            "version": "==2019.4.30"
         }
     },
     "develop": {
@@ -240,6 +240,13 @@
             ],
             "version": "==0.1.0"
         },
+        "bleach": {
+            "hashes": [
+                "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16",
+                "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa"
+            ],
+            "version": "==3.1.0"
+        },
         "certifi": {
             "hashes": [
                 "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
@@ -256,10 +263,10 @@
         },
         "commonmark": {
             "hashes": [
-                "sha256:9f6dda7876b2bb88dd784440166f4bc8e56cb2b2551264051123bacb0b6c1d8a",
-                "sha256:abcbc854e0eae5deaf52ae5e328501b78b4a0758bf98ac8bb792fce993006084"
+                "sha256:14c3df31e8c9c463377e287b2a1eefaa6019ab97b22dad36e2f32be59d61d68d",
+                "sha256:867fc5db078ede373ab811e16b6789e9d033b15ccd7296f370ca52d1ee792ce0"
             ],
-            "version": "==0.8.1"
+            "version": "==0.9.0"
         },
         "decorator": {
             "hashes": [
@@ -449,6 +456,13 @@
             ],
             "version": "==0.7.5"
         },
+        "pkginfo": {
+            "hashes": [
+                "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb",
+                "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32"
+            ],
+            "version": "==1.5.0.1"
+        },
         "prompt-toolkit": {
             "hashes": [
                 "sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780",
@@ -499,6 +513,13 @@
             ],
             "version": "==2019.1"
         },
+        "readme-renderer": {
+            "hashes": [
+                "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f",
+                "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d"
+            ],
+            "version": "==24.0"
+        },
         "recommonmark": {
             "hashes": [
                 "sha256:a520b8d25071a51ae23a27cf6252f2fe387f51bdc913390d83b2b50617f5bb48",
@@ -514,6 +535,13 @@
             ],
             "version": "==2.21.0"
         },
+        "requests-toolbelt": {
+            "hashes": [
+                "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
+                "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
+            ],
+            "version": "==0.9.1"
+        },
         "six": {
             "hashes": [
                 "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
@@ -586,6 +614,13 @@
             ],
             "version": "==1.1.3"
         },
+        "tqdm": {
+            "hashes": [
+                "sha256:d385c95361699e5cf7622485d9b9eae2d4864b21cd5a2374a9c381ffed701021",
+                "sha256:e22977e3ebe961f72362f6ddfb9197cc531c9737aaf5f607ef09740c849ecd05"
+            ],
+            "version": "==4.31.1"
+        },
         "traitlets": {
             "hashes": [
                 "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835",
@@ -593,30 +628,37 @@
             ],
             "version": "==4.3.2"
         },
+        "twine": {
+            "hashes": [
+                "sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446",
+                "sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc"
+            ],
+            "index": "pypi",
+            "version": "==1.13.0"
+        },
         "typed-ast": {
             "hashes": [
-                "sha256:04894d268ba6eab7e093d43107869ad49e7b5ef40d1a94243ea49b352061b200",
-                "sha256:16616ece19daddc586e499a3d2f560302c11f122b9c692bc216e821ae32aa0d0",
-                "sha256:252fdae740964b2d3cdfb3f84dcb4d6247a48a6abe2579e8029ab3be3cdc026c",
-                "sha256:2af80a373af123d0b9f44941a46df67ef0ff7a60f95872412a145f4500a7fc99",
-                "sha256:2c88d0a913229a06282b285f42a31e063c3bf9071ff65c5ea4c12acb6977c6a7",
-                "sha256:2ea99c029ebd4b5a308d915cc7fb95b8e1201d60b065450d5d26deb65d3f2bc1",
-                "sha256:3d2e3ab175fc097d2a51c7a0d3fda442f35ebcc93bb1d7bd9b95ad893e44c04d",
-                "sha256:4766dd695548a15ee766927bf883fb90c6ac8321be5a60c141f18628fb7f8da8",
-                "sha256:56b6978798502ef66625a2e0f80cf923da64e328da8bbe16c1ff928c70c873de",
-                "sha256:5cddb6f8bce14325b2863f9d5ac5c51e07b71b462361fd815d1d7706d3a9d682",
-                "sha256:644ee788222d81555af543b70a1098f2025db38eaa99226f3a75a6854924d4db",
-                "sha256:64cf762049fc4775efe6b27161467e76d0ba145862802a65eefc8879086fc6f8",
-                "sha256:68c362848d9fb71d3c3e5f43c09974a0ae319144634e7a47db62f0f2a54a7fa7",
-                "sha256:6c1f3c6f6635e611d58e467bf4371883568f0de9ccc4606f17048142dec14a1f",
-                "sha256:b213d4a02eec4ddf622f4d2fbc539f062af3788d1f332f028a2e19c42da53f15",
-                "sha256:bb27d4e7805a7de0e35bd0cb1411bc85f807968b2b0539597a49a23b00a622ae",
-                "sha256:c9d414512eaa417aadae7758bc118868cd2396b0e6138c1dd4fda96679c079d3",
-                "sha256:f0937165d1e25477b01081c4763d2d9cdc3b18af69cb259dd4f640c9b900fe5e",
-                "sha256:fb96a6e2c11059ecf84e6741a319f93f683e440e341d4489c9b161eca251cf2a",
-                "sha256:fc71d2d6ae56a091a8d94f33ec9d0f2001d1cb1db423d8b4355debfe9ce689b7"
-            ],
-            "version": "==1.3.4"
+                "sha256:132eae51d6ef3ff4a8c47c393a4ef5ebf0d1aecc96880eb5d6c8ceab7017cc9b",
+                "sha256:18141c1484ab8784006c839be8b985cfc82a2e9725837b0ecfa0203f71c4e39d",
+                "sha256:2baf617f5bbbfe73fd8846463f5aeafc912b5ee247f410700245d68525ec584a",
+                "sha256:3d90063f2cbbe39177e9b4d888e45777012652d6110156845b828908c51ae462",
+                "sha256:4304b2218b842d610aa1a1d87e1dc9559597969acc62ce717ee4dfeaa44d7eee",
+                "sha256:4983ede548ffc3541bae49a82675996497348e55bafd1554dc4e4a5d6eda541a",
+                "sha256:5315f4509c1476718a4825f45a203b82d7fdf2a6f5f0c8f166435975b1c9f7d4",
+                "sha256:6cdfb1b49d5345f7c2b90d638822d16ba62dc82f7616e9b4caa10b72f3f16649",
+                "sha256:7b325f12635598c604690efd7a0197d0b94b7d7778498e76e0710cd582fd1c7a",
+                "sha256:8d3b0e3b8626615826f9a626548057c5275a9733512b137984a68ba1598d3d2f",
+                "sha256:8f8631160c79f53081bd23446525db0bc4c5616f78d04021e6e434b286493fd7",
+                "sha256:912de10965f3dc89da23936f1cc4ed60764f712e5fa603a09dd904f88c996760",
+                "sha256:b010c07b975fe853c65d7bbe9d4ac62f1c69086750a574f6292597763781ba18",
+                "sha256:c908c10505904c48081a5415a1e295d8403e353e0c14c42b6d67f8f97fae6616",
+                "sha256:c94dd3807c0c0610f7c76f078119f4ea48235a953512752b9175f9f98f5ae2bd",
+                "sha256:ce65dee7594a84c466e79d7fb7d3303e7295d16a83c22c7c4037071b059e2c21",
+                "sha256:eaa9cfcb221a8a4c2889be6f93da141ac777eb8819f077e1d09fb12d00a09a93",
+                "sha256:f3376bc31bad66d46d44b4e6522c5c21976bf9bca4ef5987bb2bf727f4506cbb",
+                "sha256:f9202fa138544e13a4ec1a6792c35834250a85958fde1251b6a22e07d1260ae7"
+            ],
+            "version": "==1.3.5"
         },
         "typing-extensions": {
             "hashes": [
@@ -628,10 +670,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0",
-                "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3"
+                "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4",
+                "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb"
             ],
-            "version": "==1.24.2"
+            "version": "==1.24.3"
         },
         "wcwidth": {
             "hashes": [
@@ -639,6 +681,13 @@
                 "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
             ],
             "version": "==0.1.7"
+        },
+        "webencodings": {
+            "hashes": [
+                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
+                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+            ],
+            "version": "==0.5.1"
         }
     }
 }

archivebox/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.2

archivebox/config/__init__.py

@@ -44,10 +44,19 @@ CONFIG_DEFAULTS: Dict[str, ConfigDefaultDict] = {
         'TIMEOUT':                  {'type': int,   'default': 60},
         'MEDIA_TIMEOUT':            {'type': int,   'default': 3600},
         'OUTPUT_PERMISSIONS':       {'type': str,   'default': '755'},
-        'FOOTER_INFO':              {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
         'URL_BLACKLIST':            {'type': str,   'default': None},
     },
 
+    'SERVER_CONFIG': {
+        'SECRET_KEY':               {'type': str,   'default': None},
+        'ALLOWED_HOSTS':            {'type': str,   'default': '*'},
+        'DEBUG':                    {'type': bool,  'default': False},
+        'PUBLIC_INDEX':             {'type': bool,  'default': True},
+        'PUBLIC_SNAPSHOTS':         {'type': bool,  'default': True},
+        'FOOTER_INFO':              {'type': str,   'default': 'Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.'},
+        'ACTIVE_THEME':             {'type': str,   'default': 'default'},
+    },
+
     'ARCHIVE_METHOD_TOGGLES': {
         'SAVE_TITLE':               {'type': bool,  'default': True, 'aliases': ('FETCH_TITLE',)},
         'SAVE_FAVICON':             {'type': bool,  'default': True, 'aliases': ('FETCH_FAVICON',)},
@@ -313,9 +322,6 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
         with open(config_path, 'w+') as f:
             f.write(CONFIG_HEADER)
 
-    if not config:
-        return {}
-
     config_file = ConfigParser()
     config_file.optionxform = str
     config_file.read(config_path)
@@ -336,6 +342,21 @@ def write_config_file(config: Dict[str, str], out_dir: str=None) -> ConfigDict:
 
             config_file[section] = {**existing_config, key: val}
 
+        # always make sure there's a SECRET_KEY defined for Django
+        existing_secret_key = None
+        if 'SERVER_CONFIG' in config_file and 'SECRET_KEY' in config_file['SERVER_CONFIG']:
+            existing_secret_key = config_file['SERVER_CONFIG']['SECRET_KEY']
+
+        if (not existing_secret_key) or ('not a valid secret' in existing_secret_key):
+            from django.utils.crypto import get_random_string
+            chars = 'abcdefghijklmnopqrstuvwxyz0123456789-_+!.'
+            random_secret_key = get_random_string(50, chars)
+            if 'SERVER_CONFIG' in config_file:
+                config_file['SERVER_CONFIG']['SECRET_KEY'] = random_secret_key
+            else:
+                config_file['SERVER_CONFIG'] = {'SECRET_KEY': random_secret_key}
+
+        f.write(CONFIG_HEADER)
         config_file.write(f)
 
     try:

archivebox/config/stubs.py

@@ -22,9 +22,16 @@ class ConfigDict(BaseConfig, total=False):
     TIMEOUT: int
     MEDIA_TIMEOUT: int
     OUTPUT_PERMISSIONS: str
-    FOOTER_INFO: str
     URL_BLACKLIST: Optional[str]
 
+    SECRET_KEY: str
+    ALLOWED_HOSTS: str
+    DEBUG: bool
+    PUBLIC_INDEX: bool
+    PUBLIC_SNAPSHOTS: bool
+    FOOTER_INFO: str
+    ACTIVE_THEME: str
+
     SAVE_TITLE: bool
     SAVE_FAVICON: bool
     SAVE_WGET: bool

archivebox/core/settings.py

@@ -3,26 +3,25 @@ __package__ = 'archivebox.core'
 import os
 import sys
 
-SECRET_KEY = '---------------- not a valid secret key ! ----------------'
-DEBUG = os.getenv('DEBUG', 'False').lower() == 'true'
-ALLOWED_HOSTS = ['*']
 
-REPO_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), os.path.pardir, os.path.pardir))
-OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR', os.curdir))
-ARCHIVE_DIR = os.path.join(OUTPUT_DIR, 'archive')
-DATABASE_FILE = os.path.join(OUTPUT_DIR, 'index.sqlite3')
+from ..config import (
+    OUTPUT_DIR,
+    SECRET_KEY,
+    DEBUG,
+    ALLOWED_HOSTS,
+    PYTHON_DIR,
+    ACTIVE_THEME,
+    SQL_INDEX_FILENAME,
+)
 
-ACTIVE_THEME = 'default'
 
+ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
 
-APPEND_SLASH = True
-
 INSTALLED_APPS = [
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
-    # 'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.admin',
     'django.contrib.staticfiles',
@@ -40,17 +39,17 @@ MIDDLEWARE = [
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
-    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
 ROOT_URLCONF = 'core.urls'
+APPEND_SLASH = True
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         'DIRS': [
-            os.path.join(REPO_DIR, 'themes', ACTIVE_THEME),
-            os.path.join(REPO_DIR, 'themes', 'default'),
-            os.path.join(REPO_DIR, 'themes'),
+            os.path.join(PYTHON_DIR, 'themes', ACTIVE_THEME),
+            os.path.join(PYTHON_DIR, 'themes', 'default'),
+            os.path.join(PYTHON_DIR, 'themes'),
         ],
         'APP_DIRS': True,
         'OPTIONS': {
@@ -69,7 +68,7 @@ WSGI_APPLICATION = 'core.wsgi.application'
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': DATABASE_FILE,
+        'NAME': os.path.join(OUTPUT_DIR, SQL_INDEX_FILENAME),
     }
 }
 
@@ -104,7 +103,7 @@ SHELL_PLUS_PRINT_SQL = False
 IPYTHON_ARGUMENTS = ['--no-confirm-exit', '--no-banner']
 IPYTHON_KERNEL_DISPLAY_NAME = 'ArchiveBox Django Shell'
 if IS_SHELL:
-    os.environ['PYTHONSTARTUP'] = os.path.join(REPO_DIR, 'core', 'welcome_message.py')
+    os.environ['PYTHONSTARTUP'] = os.path.join(PYTHON_DIR, 'core', 'welcome_message.py')
 
 
 LANGUAGE_CODE = 'en-us'
@@ -118,11 +117,7 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
 STATIC_URL = '/static/'
 STATICFILES_DIRS = [
-    os.path.join(REPO_DIR, 'themes', ACTIVE_THEME, 'static'),
-    os.path.join(REPO_DIR, 'themes', 'default', 'static'),
-    os.path.join(REPO_DIR, 'themes', 'static'),
+    os.path.join(PYTHON_DIR, 'themes', ACTIVE_THEME, 'static'),
+    os.path.join(PYTHON_DIR, 'themes', 'default', 'static'),
+    os.path.join(PYTHON_DIR, 'themes', 'static'),
 ]
-
-SERVE_STATIC = True
-
-

archivebox/core/urls.py

@@ -22,8 +22,14 @@ urlpatterns = [
     path('add/', AddLinks.as_view(), name='AddLinks'),
     
     path('static/<path>', views.serve),
+    
+    path('accounts/login/', RedirectView.as_view(url='/admin/login/')),
+    path('accounts/logout/', RedirectView.as_view(url='/admin/logout/')),
+
     path('accounts/', include('django.contrib.auth.urls')),
     path('admin/', admin.site.urls),
+    
+
     path('', MainIndex.as_view(), name='Home'),
 ]
 

archivebox/core/views.py

@@ -4,11 +4,18 @@ from django.shortcuts import render, redirect
 
 from django.http import HttpResponse
 from django.views import View, static
+from django.conf import settings
 
 from core.models import Snapshot
 
 from ..index import load_main_index, load_main_index_meta
-from ..config import OUTPUT_DIR, VERSION, FOOTER_INFO
+from ..config import (
+    OUTPUT_DIR,
+    VERSION,
+    FOOTER_INFO,
+    PUBLIC_INDEX,
+    PUBLIC_SNAPSHOTS,
+)
 from ..util import base_url
 
 
@@ -16,6 +23,9 @@ class MainIndex(View):
     template = 'main_index.html'
 
     def get(self, request):
+        if not request.user.is_authenticated and not PUBLIC_INDEX:
+            return redirect(f'/admin/login/?next={request.path}')
+
         all_links = load_main_index(out_dir=OUTPUT_DIR)
         meta_info = load_main_index_meta(out_dir=OUTPUT_DIR)
 
@@ -34,6 +44,9 @@ class AddLinks(View):
     template = 'add_links.html'
 
     def get(self, request):
+        if not request.user.is_authenticated and not PUBLIC_INDEX:
+            return redirect(f'/admin/login/?next={request.path}')
+
         context = {}
 
         return render(template_name=self.template, request=request, context=context)
@@ -54,6 +67,9 @@ class LinkDetails(View):
         if '/' not in path:
             return redirect(f'{path}/index.html')
 
+        if not request.user.is_authenticated and not PUBLIC_SNAPSHOTS:
+            return redirect(f'/admin/login/?next={request.path}')
+
         try:
             slug, archivefile = path.split('/', 1)
         except (IndexError, ValueError):

archivebox/index/schema.py

@@ -59,6 +59,7 @@ class ArchiveResult:
         }
         info['start_ts'] = parse_date(info['start_ts'])
         info['end_ts'] = parse_date(info['end_ts'])
+        info['cmd_version'] = info.get('cmd_version')
         return cls(**info)
 
     def to_dict(self, *keys) -> dict:

archivebox/main.py

@@ -292,14 +292,14 @@ def init(force: bool=False, out_dir: str=OUTPUT_DIR) -> None:
     
     setup_django(out_dir, check_db=False)
     from django.conf import settings
-    assert settings.DATABASE_FILE == os.path.join(out_dir, SQL_INDEX_FILENAME)
-    print(f'    √ {settings.DATABASE_FILE}')
+    DATABASE_FILE = os.path.join(out_dir, SQL_INDEX_FILENAME)
+    print(f'    √ {DATABASE_FILE}')
     print()
     for migration_line in apply_migrations(out_dir):
         print(f'    {migration_line}')
 
 
-    assert os.path.exists(settings.DATABASE_FILE)
+    assert os.path.exists(DATABASE_FILE)
     
     # from django.contrib.auth.models import User
     # if IS_TTY and not User.objects.filter(is_superuser=True).exists():

archivebox/manage.py

@@ -3,6 +3,20 @@ import os
 import sys
 
 if __name__ == '__main__':
+    # if you're a developer working on archivebox, still prefer the archivebox
+    # versions of ./manage.py commands whenever possible. When that's not possible
+    # (e.g. makemigrations), you can comment out this check temporarily
+
+    print("[X] Don't run ./manage.py directly, use the archivebox CLI instead e.g.:")
+    print('    archivebox manage createsuperuser')
+    print()
+    print('    Hint: Use these archivebox commands instead of the ./manage.py equivalents:')
+    print('        archivebox init          (migrates the databse to latest version)')
+    print('        archivebox server        (runs the Django web server)')
+    print('        archivebox shell         (opens an iPython Django shell with all models imported)')
+    print('        archivebox manage [cmd]  (any other management commands)')
+    raise SystemExit(2)
+
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
     try:
         from django.core.management import execute_from_command_line

archivebox/themes/default/main_index.html

@@ -190,7 +190,7 @@
                     </div>
                     <div class="col-sm-10" style="text-align: right">
                         <a href="/add/">Add Links</a> &nbsp; | &nbsp; 
-                        <a href="/admin/core/page/">Admin</a> &nbsp; | &nbsp; 
+                        <a href="/admin/core/snapshot/">Admin</a> &nbsp; | &nbsp; 
                         <a href="https://github.com/pirate/ArchiveBox/wiki">Docs</a>
                     </div>
                 </div>

archivebox/util.py

@@ -10,6 +10,7 @@ from urllib.request import Request, urlopen
 from urllib.parse import urlparse, quote, unquote
 from html import escape, unescape
 from datetime import datetime
+from dateutil import parser as dateparser
 
 from base32_crockford import encode as base32_encode         # type: ignore
 import json as pyjson
@@ -140,51 +141,8 @@ def parse_date(date: Any) -> Optional[datetime]:
         date = str(date)
 
     if isinstance(date, str):
-        if date.replace('.', '').isdigit():
-            # this is a brittle attempt at unix timestamp parsing (which is
-            # notoriously hard to do). It may lead to dates being off by
-            # anything from hours to decades, depending on which app, OS,
-            # and sytem time configuration was used for the original timestamp
-            # more info: https://github.com/pirate/ArchiveBox/issues/119
-
-            # Note: always always always store the original timestamp string
-            # somewhere indepentendly of the parsed datetime, so that later
-            # bugs dont repeatedly misparse and rewrite increasingly worse dates.
-            # the correct date can always be re-derived from the timestamp str
-            timestamp = float(date)
-
-            EARLIEST_POSSIBLE = 473403600.0  # 1985
-            LATEST_POSSIBLE = 1735707600.0   # 2025
-
-            if EARLIEST_POSSIBLE < timestamp < LATEST_POSSIBLE:
-                # number is seconds
-                return datetime.fromtimestamp(timestamp)
-                
-            elif EARLIEST_POSSIBLE * 1000 < timestamp < LATEST_POSSIBLE * 1000:
-                # number is milliseconds
-                return datetime.fromtimestamp(timestamp / 1000)
-
-            elif EARLIEST_POSSIBLE * 1000*1000 < timestamp < LATEST_POSSIBLE * 1000*1000:
-                # number is microseconds
-                return datetime.fromtimestamp(timestamp / (1000*1000))
-
-            else:
-                # continue to the end and raise a parsing failed error.
-                # we dont want to even attempt parsing timestamp strings that
-                # arent within these ranges
-                pass
-
-        if '-' in date:
-            # 2019-04-07T05:44:39.227520
-            try:
-                return datetime.fromisoformat(date)
-            except Exception:
-                pass
-            try:
-                return datetime.strptime(date, '%Y-%m-%d %H:%M')
-            except Exception:
-                pass
-    
+        return dateparser.parse(date)
+
     raise ValueError('Tried to parse invalid date! {}'.format(date))
 
 

etc/ArchiveBox.conf.default

@@ -1,6 +1,6 @@
 # This is the example default configiration file for ArchiveBox.
 # 
-# Copy example config from here into your project's ArchiveBox.conf file,
+# Copy lines from here into your project's ArchiveBox.conf file and uncomment,
 # DO NOT EDIT THIS FILE DIRECTLY!
 #
 # See the list of all the possible options. documentation, and examples here:
@@ -11,10 +11,17 @@
 # ONLY_NEW = False
 # TIMEOUT = 60
 # MEDIA_TIMEOUT = 3600
-# ACTIVE_THEME = default
-# FOOTER_INFO = Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.
 # URL_BLACKLIST = (://(.*\.)?facebook\.com)|(://(.*\.)?ebay\.com)|(.*\.exe$)
 
+[SERVER_CONFIG]
+# SECRET_KEY = ---------------- not a valid secret key ! ----------------
+# DEBUG = False
+# PUBLIC_INDEX = True
+# PUBLIC_SNAPSHOTS = True
+# FOOTER_INFO = Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.
+# ACTIVE_THEME = default
+
+
 [ARCHIVE_METHOD_TOGGLES]
 # SAVE_TITLE = True
 # SAVE_FAVICON = True

setup.py

@@ -1,40 +1,34 @@
 import os
 import setuptools
 
-with open("README.md", "r") as fh:
-    long_description = fh.read()
+BASE_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
+PYTHON_DIR = os.path.join(BASE_DIR, 'archivebox')
 
+with open('README.md', "r") as f:
+    README = f.read()
 
-script_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
+with open(os.path.join(PYTHON_DIR, 'VERSION'), 'r') as f:
+    VERSION = f.read().strip()
 
-VERSION = open(os.path.join(script_dir, 'archivebox', 'VERSION'), 'r').read().strip()
-try:
-    GIT_HEAD = open(os.path.join(script_dir, '.git', 'HEAD'), 'r').read().strip().split(': ')[1]
-    GIT_SHA = open(os.path.join(script_dir, '.git', GIT_HEAD), 'r').read().strip()[:9]
-    PYPI_VERSION = "{}+{}".format(VERSION, GIT_SHA)
-except:
-    PYPI_VERSION = VERSION
-
-with open(os.path.join(script_dir, 'archivebox', 'VERSION'), 'w+') as f:
-    f.write(PYPI_VERSION)
 
 setuptools.setup(
     name="archivebox",
-    version=PYPI_VERSION,
+    version=VERSION,
     author="Nick Sweeting",
     author_email="[email protected]",
     description="The self-hosted internet archive.",
-    long_description=long_description,
+    long_description=README,
     long_description_content_type="text/markdown",
     url="https://github.com/pirate/ArchiveBox",
+    license='MIT',
     project_urls={
-        'Documentation': 'https://github.com/pirate/ArchiveBox/Wiki',
-        'Community': 'https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community',
-        'Source': 'https://github.com/pirate/ArchiveBox',
-        'Bug Tracker': 'https://github.com/pirate/ArchiveBox/issues',
-        'Roadmap': 'https://github.com/pirate/ArchiveBox/wiki/Roadmap',
+        'Donate': 'https://github.com/pirate/ArchiveBox/wiki/Donations',
         'Changelog': 'https://github.com/pirate/ArchiveBox/wiki/Changelog',
-        'Patreon': 'https://github.com/pirate/ArchiveBox/wiki/Donations',
+        'Roadmap': 'https://github.com/pirate/ArchiveBox/wiki/Roadmap',
+        'Bug Tracker': 'https://github.com/pirate/ArchiveBox/issues',
+        'Source': 'https://github.com/pirate/ArchiveBox',
+        'Community': 'https://github.com/pirate/ArchiveBox/wiki/Web-Archiving-Community',
+        'Documentation': 'https://github.com/pirate/ArchiveBox/Wiki',
     },
     packages=setuptools.find_packages(),
     python_requires='>=3.6',
@@ -61,19 +55,7 @@ setuptools.setup(
             'archivebox = archivebox.__main__:main',
         ],
     },
-    package_data={
-        'archivebox': [
-            # Manifest.ini must correspond 1:1 with this list
-            'VERSION',
-            'themes/*',
-            'themes/static/*',
-            'themes/admin/*'
-            'themes/default/*'
-            'themes/default/static/*'
-            'themes/legacy/*',
-            'themes/legacy/static/*',
-        ],
-    },
+    include_package_data=True,
     classifiers=[
         "Development Status :: 4 - Beta",