archival.ArchiveBox/archivebox/core/settings.py

__package__ = 'archivebox.core'

import os
import sys
import re
import logging

from pathlib import Path
from datetime import datetime
from django.utils.crypto import get_random_string

from ..config import (                                                          # noqa: F401
    DEBUG,
    IS_TTY,
    VERSION,
    IN_DOCKER,
    SECRET_KEY,
    ALLOWED_HOSTS,
    PACKAGE_DIR,
    TEMPLATES_DIR_NAME,
    SQL_INDEX_FILENAME,
    OUTPUT_DIR,
    LOGS_DIR,
)


IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]

################################################################################
### Django Core Settings
################################################################################

WSGI_APPLICATION = 'core.wsgi.application'
ROOT_URLCONF = 'core.urls'

LOGIN_URL = '/accounts/login/'
LOGOUT_REDIRECT_URL = '/'
PASSWORD_RESET_URL = '/accounts/password_reset/'
APPEND_SLASH = True

DEBUG = DEBUG or ('--debug' in sys.argv)

INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',

    'core',

    'django_extensions',
]


MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
]

AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
]


################################################################################
### Staticfile and Template Settings
################################################################################

STATIC_URL = '/static/'

STATICFILES_DIRS = [
    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
]

TEMPLATE_DIRS = [
    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
    str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': TEMPLATE_DIRS,
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]


################################################################################
### External Service Settings
################################################################################

DATABASE_FILE = Path(OUTPUT_DIR) / SQL_INDEX_FILENAME
DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(DATABASE_FILE))

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': DATABASE_NAME,
        # modified to be in-memory or sqlite3+wal by setup_django() in config.py
    }
}

EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'


################################################################################
### Security Settings
################################################################################

SECRET_KEY = SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_')

ALLOWED_HOSTS = ALLOWED_HOSTS.split(',')

SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True

CSRF_COOKIE_SECURE = False
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_DOMAIN = None
SESSION_COOKIE_AGE = 1209600  # 2 weeks
SESSION_EXPIRE_AT_BROWSER_CLOSE = False
SESSION_SAVE_EVERY_REQUEST = True

AUTH_PASSWORD_VALIDATORS = [
    {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
    {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'},
    {'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator'},
    {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
]


################################################################################
### Shell Settings
################################################################################

SHELL_PLUS = 'ipython'
SHELL_PLUS_PRINT_SQL = False
IPYTHON_ARGUMENTS = ['--no-confirm-exit', '--no-banner']
IPYTHON_KERNEL_DISPLAY_NAME = 'ArchiveBox Django Shell'
if IS_SHELL:
    os.environ['PYTHONSTARTUP'] = str(Path(PACKAGE_DIR) / 'core' / 'welcome_message.py')


################################################################################
### Internationalization & Localization Settings
################################################################################

LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = False
USE_L10N = False
USE_TZ = False

DATETIME_FORMAT = 'Y-m-d g:iA'
SHORT_DATETIME_FORMAT = 'Y-m-d h:iA'


################################################################################
### Logging Settings
################################################################################

IGNORABLE_404_URLS = [
    re.compile(r'apple-touch-icon.*\.png$'),
    re.compile(r'favicon\.ico$'),
    re.compile(r'robots\.txt$'),
    re.compile(r'.*\.(css|js)\.map$'),
]

class NoisyRequestsFilter(logging.Filter):
    def filter(self, record):
        logline = record.getMessage()

        # ignore harmless 404s for the patterns in IGNORABLE_404_URLS
        for ignorable_url_pattern in IGNORABLE_404_URLS:
            ignorable_log_pattern = re.compile(f'^"GET /.*/?{ignorable_url_pattern.pattern[:-1]} HTTP/.*" (200|30.|404) .+$', re.I | re.M)
            if ignorable_log_pattern.match(logline):
                return 0

        # ignore staticfile requests that 200 or 30*
        ignoreable_200_log_pattern = re.compile(r'"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M)
        if ignoreable_200_log_pattern.match(logline):
            return 0

        return 1

ERROR_LOG = LOGS_DIR / 'errors.log'

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
        'logfile': {
            'level': 'ERROR',
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': ERROR_LOG,
            'maxBytes': 1024 * 1024 * 25,  # 25 MB
            'backupCount': 10,
        },
    },
    'filters': {
        'noisyrequestsfilter': {
            '()': NoisyRequestsFilter,
        }
    },
    'loggers': {
        'django': {
            'handlers': ['console', 'logfile'],
            'level': 'DEBUG',
            'filters': ['noisyrequestsfilter'],
        },
        'django.server': {
            'handlers': ['console', 'logfile'],
            'level': 'DEBUG',
            'filters': ['noisyrequestsfilter'],
        }
    },
}


# log startup message to the error log
with open(ERROR_LOG, "a+") as f:
    command = ' '.join(sys.argv)
    ts = datetime.now().strftime('%Y-%m-%d__%H:%M:%S')
    f.write(f"\n> {command}; ts={ts} version={VERSION} docker={IN_DOCKER} is_tty={IS_TTY}\n")