move docstrings to main.py out of cli files

archivebox/cli/__init__.py

@@ -33,9 +33,8 @@ def list_subcommands() -> Dict[str, str]:
             subcommand = filename.replace('archivebox_', '').replace('.py', '')
             module = import_module('.archivebox_{}'.format(subcommand), __package__)
             assert is_valid_cli_module(module, subcommand)
-            COMMANDS.append((subcommand, module.__description__))  # type: ignore
+            COMMANDS.append((subcommand, module.main.__doc__))
             globals()[subcommand] = module.main
-            module.main.__doc__ = module.__description__
 
     display_order = lambda cmd: (
         display_first.index(cmd[0])
@@ -50,7 +49,7 @@ def run_subcommand(subcommand: str,
                    subcommand_args: List[str]=None,
                    stdin: Optional[IO]=None,
                    pwd: Optional[str]=None) -> None:
-    """run a given ArchiveBox subcommand with the given list of args"""
+    """Run a given ArchiveBox subcommand with the given list of args"""
 
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore

archivebox/cli/archivebox.py

@@ -3,7 +3,6 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox'
-__description__ = 'ArchiveBox: The self-hosted internet archive.'
 
 import sys
 import argparse
@@ -18,7 +17,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     subcommands = list_subcommands()
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description='ArchiveBox: The self-hosted internet archive',
         add_help=False,
     )
     group = parser.add_mutually_exclusive_group()

archivebox/cli/archivebox_add.py

@@ -2,29 +2,29 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox add'
-__description__ = 'Add a new URL or list of URLs to your archive'
 
 import sys
 import argparse
 
 from typing import List, Optional, IO
 
-from ..main import add
-from ..util import SmartFormatter, accept_stdin
+from ..main import add, docstring
 from ..config import OUTPUT_DIR, ONLY_NEW
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(add.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=add.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--update-all', #'-n',
         action='store_true',
-        default=not ONLY_NEW,
+        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
         help="Also retry previously skipped/failed links when adding new links",
     )
     parser.add_argument(

archivebox/cli/archivebox_config.py

@@ -2,22 +2,22 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox config'
-__description__ = 'Get and set your ArchiveBox project configuration values'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import config
-from ..util import SmartFormatter, accept_stdin
+from ..main import config, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(config.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=config.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )

archivebox/cli/archivebox_help.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox help'
-__description__ = 'Print the ArchiveBox help message and usage'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import help
-from ..util import reject_stdin
+from ..main import help, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(help.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=help.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_info.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox info'
-__description__ = 'Print out some info and statistics about the archive collection'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import info
+from ..main import info, docstring
 from ..config import OUTPUT_DIR
-from ..util import reject_stdin
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(info.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=info.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_init.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox init'
-__description__ = 'Initialize a new ArchiveBox collection in the current directory'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import init
-from ..util import reject_stdin
+from ..main import init, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(init.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=init.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_list.py

@@ -2,15 +2,13 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox list'
-__description__ = 'List, filter, and export information about archive entries'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import list_all
-from ..util import SmartFormatter, accept_stdin
+from ..main import list_all, docstring
 from ..config import OUTPUT_DIR
 from ..index import (
     get_indexed_folders,
@@ -24,11 +22,14 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
+from .logging import SmartFormatter, accept_stdin
 
+
+@docstring(list_all.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=list_all.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )

archivebox/cli/archivebox_manage.py

@@ -2,16 +2,16 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox manage'
-__description__ = 'Run an ArchiveBox Django management command'
 
 import sys
 
 from typing import Optional, List, IO
 
-from ..main import manage
+from ..main import manage, docstring
 from ..config import OUTPUT_DIR
 
 
+@docstring(manage.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     manage(
         args=args,

archivebox/cli/archivebox_remove.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox remove'
-__description__ = 'Remove the specified URLs from the archive.'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import remove
-from ..util import accept_stdin
+from ..main import remove, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(remove.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=remove.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--yes', # '-y',

archivebox/cli/archivebox_schedule.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox schedule'
-__description__ = 'Set ArchiveBox to regularly import URLs at specific times using cron'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import schedule
-from ..util import reject_stdin
+from ..main import schedule, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(schedule.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=schedule.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--quiet', '-q',

archivebox/cli/archivebox_server.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox server'
-__description__ = 'Run the ArchiveBox HTTP server'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import server
-from ..util import reject_stdin
+from ..main import server, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(server.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=server.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         'runserver_args',

archivebox/cli/archivebox_shell.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox shell'
-__description__ = 'Enter an interactive ArchiveBox Django shell'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import shell
+from ..main import shell, docstring
 from ..config import OUTPUT_DIR
-from ..util import reject_stdin
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(shell.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=shell.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_update.py

@@ -2,15 +2,13 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox update'
-__description__ = 'Import any new links from subscriptions and retry any previously failed/skipped links'
 
 import sys
 import argparse
 
 from typing import List, Optional, IO
 
-from ..main import update
-from ..util import SmartFormatter, accept_stdin
+from ..main import update, docstring
 from ..config import OUTPUT_DIR
 from ..index import (
     get_indexed_folders,
@@ -24,12 +22,14 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(update.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=update.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )
@@ -99,9 +99,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         nargs='*',
         type=str,
         default=None,
-        help='List only URLs matching these filter patterns.'
+        help='Update only URLs matching these filter patterns.'
     )
-    command = parser.parse_args(args)
+    command = parser.parse_args(args or ())
     filter_patterns_str = accept_stdin(stdin)
 
     update(

archivebox/cli/archivebox_version.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox version'
-__description__ = 'Print the ArchiveBox version and dependency information'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import version
-from ..util import reject_stdin
+from ..main import version, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(version.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=version.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--quiet', '-q',

archivebox/main.py

@@ -118,7 +118,10 @@ ALLOWED_IN_OUTPUT_DIR = {
     FAVICON_FILENAME,
 }
 
+@enforce_types
 def help(out_dir: str=OUTPUT_DIR) -> None:
+    """Print the ArchiveBox help message and usage"""
+
     all_subcommands = list_subcommands()
     COMMANDS_HELP_TEXT = '\n    '.join(
         f'{cmd.ljust(20)} {summary}'
@@ -182,7 +185,11 @@ def help(out_dir: str=OUTPUT_DIR) -> None:
         print('    https://github.com/pirate/ArchiveBox/wiki')
 
 
-def version(quiet: bool=False, out_dir: str=OUTPUT_DIR) -> None:
+@enforce_types
+def version(quiet: bool=False,
+            out_dir: str=OUTPUT_DIR) -> None:
+    """Print the ArchiveBox version and dependency information"""
+
     if quiet:
         print(VERSION)
     else:
@@ -191,37 +198,44 @@ def version(quiet: bool=False, out_dir: str=OUTPUT_DIR) -> None:
 
         print('{white}[i] Dependency versions:{reset}'.format(**ANSI))
         for name, dependency in DEPENDENCIES.items():
-            print_dependency_version(name, dependency)
+            print(printable_dependency_version(name, dependency))
         
         print()
         print('{white}[i] Code locations:{reset}'.format(**ANSI))
         for name, folder in CODE_LOCATIONS.items():
-            print_folder_status(name, folder)
+            print(printable_folder_status(name, folder))
 
         print()
         print('{white}[i] External locations:{reset}'.format(**ANSI))
         for name, folder in EXTERNAL_LOCATIONS.items():
-            print_folder_status(name, folder)
+            print(printable_folder_status(name, folder))
 
         print()
         print('{white}[i] Data locations:{reset}'.format(**ANSI))
         for name, folder in DATA_LOCATIONS.items():
-            print_folder_status(name, folder)
+            print(printable_folder_status(name, folder))
 
         print()
         check_dependencies()
 
 
-def run(subcommand: str, subcommand_args: Optional[List[str]], stdin: Optional[IO]=None, out_dir: str=OUTPUT_DIR) -> None:
+@enforce_types
+def run(subcommand: str,
+        subcommand_args: Optional[List[str]],
+        stdin: Optional[IO]=None,
+        out_dir: str=OUTPUT_DIR) -> None:
+    """Run a given ArchiveBox subcommand with the given list of args"""
     run_subcommand(
         subcommand=subcommand,
         subcommand_args=subcommand_args,
         stdin=stdin,
-        out_dir=out_dir,
+        pwd=out_dir,
     )
 
 
+@enforce_types
 def init(out_dir: str=OUTPUT_DIR) -> None:
+    """Initialize a new ArchiveBox collection in the current directory"""
     os.makedirs(out_dir, exist_ok=True)
 
     is_empty = not len(set(os.listdir(out_dir)) - ALLOWED_IN_OUTPUT_DIR)
@@ -364,7 +378,10 @@ def init(out_dir: str=OUTPUT_DIR) -> None:
     print('        archivebox help')
 
 
+@enforce_types
 def info(out_dir: str=OUTPUT_DIR) -> None:
+    """Print out some info and statistics about the archive collection"""
+
     check_data_folder(out_dir=out_dir)
 
     print('{green}[*] Scanning archive collection main index...{reset}'.format(**ANSI))
@@ -454,6 +471,7 @@ def add(import_str: Optional[str]=None,
         update_all: bool=not ONLY_NEW,
         index_only: bool=False,
         out_dir: str=OUTPUT_DIR) -> List[Link]:
+    """Add a new URL or list of URLs to your archive"""
 
     check_data_folder(out_dir=out_dir)
 
@@ -518,6 +536,7 @@ def remove(filter_str: Optional[str]=None,
            yes: bool=False,
            delete: bool=False,
            out_dir: str=OUTPUT_DIR) -> List[Link]:
+    """Remove the specified URLs from the archive"""
     
     check_data_folder(out_dir=out_dir)
 
@@ -586,7 +605,7 @@ def remove(filter_str: Optional[str]=None,
 
 @enforce_types
 def update(resume: Optional[float]=None,
-           only_new: bool=not ONLY_NEW,
+           only_new: bool=ONLY_NEW,
            index_only: bool=False,
            overwrite: bool=False,
            filter_patterns_str: Optional[str]=None,
@@ -596,6 +615,7 @@ def update(resume: Optional[float]=None,
            after: Optional[str]=None,
            before: Optional[str]=None,
            out_dir: str=OUTPUT_DIR) -> List[Link]:
+    """Import any new links from subscriptions and retry any previously failed/skipped links"""
 
     check_dependencies()
     check_data_folder(out_dir=out_dir)
@@ -659,8 +679,9 @@ def list_all(filter_patterns_str: Optional[str]=None,
              before: Optional[float]=None,
              sort: Optional[str]=None,
              csv: Optional[str]=None,
-             json: Optional[str]=None,
+             json: bool=False,
              out_dir: str=OUTPUT_DIR) -> Iterable[Link]:
+    """List, filter, and export information about archive entries"""
     
     check_data_folder(out_dir=out_dir)
 
@@ -756,12 +777,14 @@ def list_folders(links: List[Link],
     raise ValueError('Status not recognized.')
 
 
+@enforce_types
 def config(config_options_str: Optional[str]=None,
            config_options: Optional[List[str]]=None,
            get: bool=False,
            set: bool=False,
            reset: bool=False,
            out_dir: str=OUTPUT_DIR) -> None:
+    """Get and set your ArchiveBox project configuration values"""
 
     check_data_folder(out_dir=out_dir)
 
@@ -863,6 +886,7 @@ def schedule(add: bool=False,
              every: Optional[str]=None,
              import_path: Optional[str]=None,
              out_dir: str=OUTPUT_DIR):
+    """Set ArchiveBox to regularly import URLs at specific times using cron"""
     
     check_data_folder(out_dir=out_dir)
 
@@ -957,10 +981,13 @@ def schedule(add: bool=False,
         raise SystemExit(0)
 
 
+@enforce_types
+def server(runserver_args: Optional[List[str]]=None,
+           reload: bool=False,
+           debug: bool=False,
+           out_dir: str=OUTPUT_DIR) -> None:
+    """Run the ArchiveBox HTTP server"""
 
-
-
-def server(runserver_args: Optional[List[str]]=None, reload: bool=False, out_dir: str=OUTPUT_DIR) -> None:
     runserver_args = runserver_args or []
     check_data_folder(out_dir=out_dir)
 
@@ -982,7 +1009,10 @@ def server(runserver_args: Optional[List[str]]=None, reload: bool=False, out_dir
     call_command("runserver", *runserver_args)
 
 
+@enforce_types
 def manage(args: Optional[List[str]]=None, out_dir: str=OUTPUT_DIR) -> None:
+    """Run an ArchiveBox Django management command"""
+
     check_data_folder(out_dir=out_dir)
 
     setup_django(out_dir)