gallery-dl/gallery_dl/option.py

# -*- coding: utf-8 -*-

# Copyright 2017-2021 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

"""Command line option parsing"""

import argparse
import logging
import json
import sys
from . import job, version


class ConfigAction(argparse.Action):
    """Set argparse results as config values"""
    def __call__(self, parser, namespace, values, option_string=None):
        namespace.options.append(((), self.dest, values))


class ConfigConstAction(argparse.Action):
    """Set argparse const values as config values"""
    def __call__(self, parser, namespace, values, option_string=None):
        namespace.options.append(((), self.dest, self.const))


class AppendCommandAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        items = getattr(namespace, self.dest, None) or []
        val = self.const.copy()
        val["command"] = values
        items.append(val)
        setattr(namespace, self.dest, items)


class DeprecatedConfigConstAction(argparse.Action):
    """Set argparse const values as config values + deprecation warning"""
    def __call__(self, parser, namespace, values, option_string=None):
        print("warning: {} is deprecated. Use {} instead.".format(
            "/".join(self.option_strings), self.choices), file=sys.stderr)
        namespace.options.append(((), self.dest, self.const))


class ParseAction(argparse.Action):
    """Parse <key>=<value> options and set them as config values"""
    def __call__(self, parser, namespace, values, option_string=None):
        key, _, value = values.partition("=")
        try:
            value = json.loads(value)
        except ValueError:
            pass
        key = key.split(".")  # splitting an empty string becomes [""]
        namespace.options.append((key[:-1], key[-1], value))


class Formatter(argparse.HelpFormatter):
    """Custom HelpFormatter class to customize help output"""
    def __init__(self, *args, **kwargs):
        super().__init__(max_help_position=50, *args, **kwargs)

    def _format_action_invocation(self, action):
        opts = action.option_strings[:]
        if opts:
            if action.nargs != 0:
                args_string = self._format_args(action, "ARG")
                opts[-1] += " " + args_string
            return ', '.join(opts)
        else:
            return self._metavar_formatter(action, action.dest)(1)[0]


def build_parser():
    """Build and configure an ArgumentParser object"""
    parser = argparse.ArgumentParser(
        usage="%(prog)s [OPTION]... URL...",
        formatter_class=Formatter,
        add_help=False,
    )

    general = parser.add_argument_group("General Options")
    general.add_argument(
        "-h", "--help",
        action="help",
        help="Print this help message and exit",
    )
    general.add_argument(
        "--version",
        action="version", version=version.__version__,
        help="Print program version and exit",
    )
    general.add_argument(
        "-i", "--input-file",
        dest="inputfiles", metavar="FILE", action="append",
        help=("Download URLs found in FILE ('-' for stdin). "
              "More than one --input-file can be specified"),
    )
    general.add_argument(
        "-d", "--destination",
        dest="base-directory", metavar="PATH", action=ConfigAction,
        help="Target location for file downloads",
    )
    general.add_argument(
        "-D", "--directory",
        dest="directory", metavar="PATH",
        help="Exact location for file downloads",
    )
    general.add_argument(
        "-f", "--filename",
        dest="filename", metavar="FORMAT",
        help=("Filename format string for downloaded files "
              "('/O' for \"original\" filenames)"),
    )
    general.add_argument(
        "--cookies",
        dest="cookies", metavar="FILE", action=ConfigAction,
        help="File to load additional cookies from",
    )
    general.add_argument(
        "--proxy",
        dest="proxy", metavar="URL", action=ConfigAction,
        help="Use the specified proxy",
    )
    general.add_argument(
        "--source-address",
        dest="source-address", metavar="IP", action=ConfigAction,
        help="Client-side IP address to bind to",
    )
    general.add_argument(
        "--clear-cache",
        dest="clear_cache", metavar="MODULE",
        help="Delete cached login sessions, cookies, etc. for MODULE "
             "(ALL to delete everything)",
    )

    output = parser.add_argument_group("Output Options")
    output.add_argument(
        "-q", "--quiet",
        dest="loglevel", default=logging.INFO,
        action="store_const", const=logging.ERROR,
        help="Activate quiet mode",
    )
    output.add_argument(
        "-v", "--verbose",
        dest="loglevel",
        action="store_const", const=logging.DEBUG,
        help="Print various debugging information",
    )
    output.add_argument(
        "-g", "--get-urls",
        dest="list_urls", action="count",
        help="Print URLs instead of downloading",
    )
    output.add_argument(
        "-G", "--resolve-urls",
        dest="list_urls", action="store_const", const=128,
        help="Print URLs instead of downloading; resolve intermediary URLs",
    )
    output.add_argument(
        "-j", "--dump-json",
        dest="jobtype", action="store_const", const=job.DataJob,
        help="Print JSON information",
    )
    output.add_argument(
        "-s", "--simulate",
        dest="jobtype", action="store_const", const=job.SimulationJob,
        help="Simulate data extraction; do not download anything",
    )
    output.add_argument(
        "-E", "--extractor-info",
        dest="jobtype", action="store_const", const=job.InfoJob,
        help="Print extractor defaults and settings",
    )
    output.add_argument(
        "-K", "--list-keywords",
        dest="jobtype", action="store_const", const=job.KeywordJob,
        help=("Print a list of available keywords and example values "
              "for the given URLs"),
    )
    output.add_argument(
        "--list-modules",
        dest="list_modules", action="store_true",
        help="Print a list of available extractor modules",
    )
    output.add_argument(
        "--list-extractors",
        dest="list_extractors", action="store_true",
        help=("Print a list of extractor classes "
              "with description, (sub)category and example URL"),
    )
    output.add_argument(
        "--write-log",
        dest="logfile", metavar="FILE", action=ConfigAction,
        help="Write logging output to FILE",
    )
    output.add_argument(
        "--write-unsupported",
        dest="unsupportedfile", metavar="FILE", action=ConfigAction,
        help=("Write URLs, which get emitted by other extractors but cannot "
              "be handled, to FILE"),
    )
    output.add_argument(
        "--write-pages",
        dest="write-pages", nargs=0, action=ConfigConstAction, const=True,
        help=("Write downloaded intermediary pages to files "
              "in the current directory to debug problems"),
    )

    downloader = parser.add_argument_group("Downloader Options")
    downloader.add_argument(
        "-r", "--limit-rate",
        dest="rate", metavar="RATE", action=ConfigAction,
        help="Maximum download rate (e.g. 500k or 2.5M)",
    )
    downloader.add_argument(
        "-R", "--retries",
        dest="retries", metavar="N", type=int, action=ConfigAction,
        help=("Maximum number of retries for failed HTTP requests "
              "or -1 for infinite retries (default: 4)"),
    )
    downloader.add_argument(
        "--http-timeout",
        dest="timeout", metavar="SECONDS", type=float, action=ConfigAction,
        help="Timeout for HTTP connections (default: 30.0)",
    )
    downloader.add_argument(
        "--sleep",
        dest="sleep", metavar="SECONDS", action=ConfigAction,
        help=("Number of seconds to wait before each download. "
              "This can be either a constant value or a range "
              "(e.g. 2.7 or 2.0-3.5)"),
    )
    downloader.add_argument(
        "--sleep-request",
        dest="sleep-request", metavar="SECONDS", action=ConfigAction,
        help=("Number of seconds to wait between HTTP requests "
              "during data extraction"),
    )
    downloader.add_argument(
        "--sleep-extractor",
        dest="sleep-extractor", metavar="SECONDS", action=ConfigAction,
        help=("Number of seconds to wait before starting data extraction "
              "for an input URL"),
    )
    downloader.add_argument(
        "--filesize-min",
        dest="filesize-min", metavar="SIZE", action=ConfigAction,
        help="Do not download files smaller than SIZE (e.g. 500k or 2.5M)",
    )
    downloader.add_argument(
        "--filesize-max",
        dest="filesize-max", metavar="SIZE", action=ConfigAction,
        help="Do not download files larger than SIZE (e.g. 500k or 2.5M)",
    )
    downloader.add_argument(
        "--no-part",
        dest="part", nargs=0, action=ConfigConstAction, const=False,
        help="Do not use .part files",
    )
    downloader.add_argument(
        "--no-skip",
        dest="skip", nargs=0, action=ConfigConstAction, const=False,
        help="Do not skip downloads; overwrite existing files",
    )
    downloader.add_argument(
        "--no-mtime",
        dest="mtime", nargs=0, action=ConfigConstAction, const=False,
        help=("Do not set file modification times according to "
              "Last-Modified HTTP response headers")
    )
    downloader.add_argument(
        "--no-download",
        dest="download", nargs=0, action=ConfigConstAction, const=False,
        help=("Do not download any files")
    )
    downloader.add_argument(
        "--no-check-certificate",
        dest="verify", nargs=0, action=ConfigConstAction, const=False,
        help="Disable HTTPS certificate validation",
    )

    configuration = parser.add_argument_group("Configuration Options")
    configuration.add_argument(
        "-c", "--config",
        dest="cfgfiles", metavar="FILE", action="append",
        help="Additional configuration files",
    )
    configuration.add_argument(
        "--config-yaml",
        dest="yamlfiles", metavar="FILE", action="append",
        help=argparse.SUPPRESS,
    )
    configuration.add_argument(
        "-o", "--option",
        dest="options", metavar="OPT", action=ParseAction, default=[],
        help="Additional '<key>=<value>' option values",
    )
    configuration.add_argument(
        "--ignore-config",
        dest="load_config", action="store_false",
        help="Do not read the default configuration files",
    )

    authentication = parser.add_argument_group("Authentication Options")
    authentication.add_argument(
        "-u", "--username",
        dest="username", metavar="USER", action=ConfigAction,
        help="Username to login with",
    )
    authentication.add_argument(
        "-p", "--password",
        dest="password", metavar="PASS", action=ConfigAction,
        help="Password belonging to the given username",
    )
    authentication.add_argument(
        "--netrc",
        dest="netrc", nargs=0, action=ConfigConstAction, const=True,
        help="Enable .netrc authentication data",
    )

    selection = parser.add_argument_group("Selection Options")
    selection.add_argument(
        "--download-archive",
        dest="archive", metavar="FILE", action=ConfigAction,
        help=("Record all downloaded files in the archive file and "
              "skip downloading any file already in it"),
    )
    selection.add_argument(
        "-A", "--abort",
        dest="abort", metavar="N", type=int,
        help=("Stop current extractor run "
              "after N consecutive file downloads were skipped"),
    )
    selection.add_argument(
        "-T", "--terminate",
        dest="terminate", metavar="N", type=int,
        help=("Stop current and parent extractor runs "
              "after N consecutive file downloads were skipped"),
    )
    selection.add_argument(
        "--range",
        dest="image-range", metavar="RANGE", action=ConfigAction,
        help=("Index-range(s) specifying which images to download. "
              "For example '5-10' or '1,3-5,10-'"),
    )
    selection.add_argument(
        "--chapter-range",
        dest="chapter-range", metavar="RANGE", action=ConfigAction,
        help=("Like '--range', but applies to manga-chapters "
              "and other delegated URLs"),
    )
    selection.add_argument(
        "--filter",
        dest="image-filter", metavar="EXPR", action=ConfigAction,
        help=("Python expression controlling which images to download. "
              "Files for which the expression evaluates to False are ignored. "
              "Available keys are the filename-specific ones listed by '-K'. "
              "Example: --filter \"image_width >= 1000 and "
              "rating in ('s', 'q')\""),
    )
    selection.add_argument(
        "--chapter-filter",
        dest="chapter-filter", metavar="EXPR", action=ConfigAction,
        help=("Like '--filter', but applies to manga-chapters "
              "and other delegated URLs"),
    )

    infojson = {
        "name"    : "metadata",
        "event"   : "init",
        "filename": "info.json",
    }
    postprocessor = parser.add_argument_group("Post-processing Options")
    postprocessor.add_argument(
        "--zip",
        dest="postprocessors",
        action="append_const", const="zip",
        help="Store downloaded files in a ZIP archive",
    )
    postprocessor.add_argument(
        "--ugoira-conv",
        dest="postprocessors", action="append_const", const={
            "name"          : "ugoira",
            "ffmpeg-args"   : ("-c:v", "libvpx", "-crf", "4", "-b:v", "5000k"),
            "ffmpeg-twopass": True,
            "whitelist"     : ("pixiv", "danbooru"),
        },
        help="Convert Pixiv Ugoira to WebM (requires FFmpeg)",
    )
    postprocessor.add_argument(
        "--ugoira-conv-lossless",
        dest="postprocessors", action="append_const", const={
            "name"          : "ugoira",
            "ffmpeg-args"   : ("-c:v", "libvpx-vp9", "-lossless", "1",
                               "-pix_fmt", "yuv420p"),
            "ffmpeg-twopass": False,
            "whitelist"     : ("pixiv", "danbooru"),
        },
        help="Convert Pixiv Ugoira to WebM in VP9 lossless mode",
    )
    postprocessor.add_argument(
        "--write-metadata",
        dest="postprocessors",
        action="append_const", const="metadata",
        help="Write metadata to separate JSON files",
    )
    postprocessor.add_argument(
        "--write-info-json",
        dest="postprocessors",
        action="append_const", const=infojson,
        help="Write gallery metadata to a info.json file",
    )
    postprocessor.add_argument(
        "--write-infojson",
        dest="postprocessors",
        action="append_const", const=infojson,
        help=argparse.SUPPRESS,
    )
    postprocessor.add_argument(
        "--write-tags",
        dest="postprocessors",
        action="append_const", const={"name": "metadata", "mode": "tags"},
        help="Write image tags to separate text files",
    )
    postprocessor.add_argument(
        "--mtime-from-date",
        dest="postprocessors",
        action="append_const", const="mtime",
        help="Set file modification times according to 'date' metadata",
    )
    postprocessor.add_argument(
        "--exec",
        dest="postprocessors", metavar="CMD",
        action=AppendCommandAction, const={"name": "exec"},
        help=("Execute CMD for each downloaded file. "
              "Example: --exec 'convert {} {}.png && rm {}'"),
    )
    postprocessor.add_argument(
        "--exec-after",
        dest="postprocessors", metavar="CMD",
        action=AppendCommandAction, const={
            "name": "exec", "event": "finalize"},
        help=("Execute CMD after all files were downloaded successfully. "
              "Example: --exec-after 'cd {} && convert * ../doc.pdf'"),
    )
    postprocessor.add_argument(
        "-P", "--postprocessor",
        dest="postprocessors", metavar="NAME", action="append",
        help="Activate the specified post processor",
    )

    parser.add_argument(
        "urls",
        metavar="URL", nargs="*",
        help=argparse.SUPPRESS,
    )

    return parser