1
0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-22 18:53:21 +01:00
gallery-dl/gallery_dl/option.py

734 lines
26 KiB
Python

# -*- coding: utf-8 -*-
# Copyright 2017-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Command line option parsing"""
import argparse
import logging
import sys
from . import job, util, version
class ConfigAction(argparse.Action):
"""Set argparse results as config values"""
def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((), self.dest, values))
class ConfigConstAction(argparse.Action):
"""Set argparse const values as config values"""
def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((), self.dest, self.const))
class AppendCommandAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest, None) or []
val = self.const.copy()
val["command"] = values
items.append(val)
setattr(namespace, self.dest, items)
class DeprecatedConfigConstAction(argparse.Action):
"""Set argparse const values as config values + deprecation warning"""
def __call__(self, parser, namespace, values, option_string=None):
sys.stderr.write(
"warning: {} is deprecated. Use {} instead.\n".format(
"/".join(self.option_strings), self.choices))
namespace.options.append(((), self.dest, self.const))
class ConfigParseAction(argparse.Action):
"""Parse KEY=VALUE config options"""
def __call__(self, parser, namespace, values, option_string=None):
key, value = _parse_option(values)
key = key.split(".") # splitting an empty string becomes [""]
namespace.options.append((key[:-1], key[-1], value))
class PPParseAction(argparse.Action):
"""Parse KEY=VALUE post processor options"""
def __call__(self, parser, namespace, values, option_string=None):
key, value = _parse_option(values)
namespace.options_pp[key] = value
class InputfileAction(argparse.Action):
"""Collect input files"""
def __call__(self, parser, namespace, value, option_string=None):
namespace.input_files.append((value, self.const))
class MtimeAction(argparse.Action):
"""Configure mtime post processors"""
def __call__(self, parser, namespace, value, option_string=None):
namespace.postprocessors.append({
"name": "mtime",
"value": "{" + (self.const or value) + "}",
})
class RenameAction(argparse.Action):
"""Configure rename post processors"""
def __call__(self, parser, namespace, value, option_string=None):
if self.const:
namespace.postprocessors.append({
"name": "rename",
"to" : value,
})
else:
namespace.postprocessors.append({
"name": "rename",
"from": value,
})
class UgoiraAction(argparse.Action):
"""Configure ugoira post processors"""
def __call__(self, parser, namespace, value, option_string=None):
if self.const:
value = self.const
else:
value = value.strip().lower()
if value in ("webm", "vp9"):
pp = {
"extension" : "webm",
"ffmpeg-args" : ("-c:v", "libvpx-vp9",
"-crf", "12",
"-b:v", "0", "-an"),
}
elif value == "vp9-lossless":
pp = {
"extension" : "webm",
"ffmpeg-args" : ("-c:v", "libvpx-vp9",
"-lossless", "1",
"-pix_fmt", "yuv420p", "-an"),
}
elif value == "vp8":
pp = {
"extension" : "webm",
"ffmpeg-args" : ("-c:v", "libvpx",
"-crf", "4",
"-b:v", "5000k", "-an"),
}
elif value == "mp4":
pp = {
"extension" : "mp4",
"ffmpeg-args" : ("-c:v", "libx264", "-an", "-b:v", "5M"),
"libx264-prevent-odd": True,
}
elif value == "gif":
pp = {
"extension" : "gif",
"ffmpeg-args" : ("-filter_complex", "[0:v] split [a][b];"
"[a] palettegen [p];[b][p] paletteuse"),
"repeat-last-frame": False,
}
elif value in ("mkv", "copy"):
pp = {
"extension" : "mkv",
"ffmpeg-args" : ("-c:v", "copy"),
"repeat-last-frame": False,
}
else:
parser.error("Unsupported Ugoira format '{}'".format(value))
pp["name"] = "ugoira"
pp["whitelist"] = ("pixiv", "danbooru")
namespace.options.append(((), "ugoira", True))
namespace.postprocessors.append(pp)
class Formatter(argparse.HelpFormatter):
"""Custom HelpFormatter class to customize help output"""
def __init__(self, prog):
argparse.HelpFormatter.__init__(self, prog, max_help_position=30)
def _format_action_invocation(self, action, join=", ".join):
opts = action.option_strings
if action.metavar:
opts = opts.copy()
opts[-1] += " " + action.metavar
return join(opts)
def _parse_option(opt):
key, _, value = opt.partition("=")
try:
value = util.json_loads(value)
except ValueError:
pass
return key, value
def build_parser():
"""Build and configure an ArgumentParser object"""
parser = argparse.ArgumentParser(
usage="%(prog)s [OPTION]... URL...",
formatter_class=Formatter,
add_help=False,
)
general = parser.add_argument_group("General Options")
general.add_argument(
"-h", "--help",
action="help",
help="Print this help message and exit",
)
general.add_argument(
"--version",
action="version", version=version.__version__,
help="Print program version and exit",
)
general.add_argument(
"-f", "--filename",
dest="filename", metavar="FORMAT",
help=("Filename format string for downloaded files "
"('/O' for \"original\" filenames)"),
)
general.add_argument(
"-d", "--destination",
dest="base-directory", metavar="PATH", action=ConfigAction,
help="Target location for file downloads",
)
general.add_argument(
"-D", "--directory",
dest="directory", metavar="PATH",
help="Exact location for file downloads",
)
general.add_argument(
"-X", "--extractors",
dest="extractor_sources", metavar="PATH", action="append",
help="Load external extractors from PATH",
)
general.add_argument(
"--user-agent",
dest="user-agent", metavar="UA", action=ConfigAction,
help="User-Agent request header",
)
general.add_argument(
"--clear-cache",
dest="clear_cache", metavar="MODULE",
help="Delete cached login sessions, cookies, etc. for MODULE "
"(ALL to delete everything)",
)
update = parser.add_argument_group("Update Options")
if util.EXECUTABLE:
update.add_argument(
"-U", "--update",
dest="update", action="store_const", const="latest",
help="Update to the latest version",
)
update.add_argument(
"--update-to",
dest="update", metavar="CHANNEL[@TAG]",
help=("Switch to a dfferent release channel (stable or dev) "
"or upgrade/downgrade to a specific version"),
)
update.add_argument(
"--update-check",
dest="update", action="store_const", const="check",
help="Check if a newer version is available",
)
else:
update.add_argument(
"-U", "--update-check",
dest="update", action="store_const", const="check",
help="Check if a newer version is available",
)
input = parser.add_argument_group("Input Options")
input.add_argument(
"urls",
metavar="URL", nargs="*",
help=argparse.SUPPRESS,
)
input.add_argument(
"-i", "--input-file",
dest="input_files", metavar="FILE", action=InputfileAction, const=None,
default=[],
help=("Download URLs found in FILE ('-' for stdin). "
"More than one --input-file can be specified"),
)
input.add_argument(
"-I", "--input-file-comment",
dest="input_files", metavar="FILE", action=InputfileAction, const="c",
help=("Download URLs found in FILE. "
"Comment them out after they were downloaded successfully."),
)
input.add_argument(
"-x", "--input-file-delete",
dest="input_files", metavar="FILE", action=InputfileAction, const="d",
help=("Download URLs found in FILE. "
"Delete them after they were downloaded successfully."),
)
input.add_argument(
"--no-input",
dest="input", nargs=0, action=ConfigConstAction, const=False,
help=("Do not prompt for passwords/tokens"),
)
output = parser.add_argument_group("Output Options")
output.add_argument(
"-q", "--quiet",
dest="loglevel", default=logging.INFO,
action="store_const", const=logging.ERROR,
help="Activate quiet mode",
)
output.add_argument(
"-w", "--warning",
dest="loglevel",
action="store_const", const=logging.WARNING,
help="Print only warnings and errors",
)
output.add_argument(
"-v", "--verbose",
dest="loglevel",
action="store_const", const=logging.DEBUG,
help="Print various debugging information",
)
output.add_argument(
"-g", "--get-urls",
dest="list_urls", action="count",
help="Print URLs instead of downloading",
)
output.add_argument(
"-G", "--resolve-urls",
dest="list_urls", action="store_const", const=128,
help="Print URLs instead of downloading; resolve intermediary URLs",
)
output.add_argument(
"-j", "--dump-json",
dest="dump_json", action="count",
help="Print JSON information",
)
output.add_argument(
"-J", "--resolve-json",
dest="dump_json", action="store_const", const=128,
help="Print JSON information; resolve intermediary URLs",
)
output.add_argument(
"-s", "--simulate",
dest="jobtype", action="store_const", const=job.SimulationJob,
help="Simulate data extraction; do not download anything",
)
output.add_argument(
"-E", "--extractor-info",
dest="jobtype", action="store_const", const=job.InfoJob,
help="Print extractor defaults and settings",
)
output.add_argument(
"-K", "--list-keywords",
dest="jobtype", action="store_const", const=job.KeywordJob,
help=("Print a list of available keywords and example values "
"for the given URLs"),
)
output.add_argument(
"-e", "--error-file",
dest="errorfile", metavar="FILE", action=ConfigAction,
help="Add input URLs which returned an error to FILE",
)
output.add_argument(
"--list-modules",
dest="list_modules", action="store_true",
help="Print a list of available extractor modules",
)
output.add_argument(
"--list-extractors",
dest="list_extractors", action="store_true",
help=("Print a list of extractor classes "
"with description, (sub)category and example URL"),
)
output.add_argument(
"--write-log",
dest="logfile", metavar="FILE", action=ConfigAction,
help="Write logging output to FILE",
)
output.add_argument(
"--write-unsupported",
dest="unsupportedfile", metavar="FILE", action=ConfigAction,
help=("Write URLs, which get emitted by other extractors but cannot "
"be handled, to FILE"),
)
output.add_argument(
"--write-pages",
dest="write-pages", nargs=0, action=ConfigConstAction, const=True,
help=("Write downloaded intermediary pages to files "
"in the current directory to debug problems"),
)
output.add_argument(
"--print-traffic",
dest="print_traffic", action="store_true",
help=("Display sent and read HTTP traffic"),
)
output.add_argument(
"--no-colors",
dest="colors", action="store_false",
help=("Do not emit ANSI color codes in output"),
)
networking = parser.add_argument_group("Networking Options")
networking.add_argument(
"-R", "--retries",
dest="retries", metavar="N", type=int, action=ConfigAction,
help=("Maximum number of retries for failed HTTP requests "
"or -1 for infinite retries (default: 4)"),
)
networking.add_argument(
"--http-timeout",
dest="timeout", metavar="SECONDS", type=float, action=ConfigAction,
help="Timeout for HTTP connections (default: 30.0)",
)
networking.add_argument(
"--proxy",
dest="proxy", metavar="URL", action=ConfigAction,
help="Use the specified proxy",
)
networking.add_argument(
"--source-address",
dest="source-address", metavar="IP", action=ConfigAction,
help="Client-side IP address to bind to",
)
networking.add_argument(
"--no-check-certificate",
dest="verify", nargs=0, action=ConfigConstAction, const=False,
help="Disable HTTPS certificate validation",
)
downloader = parser.add_argument_group("Downloader Options")
downloader.add_argument(
"-r", "--limit-rate",
dest="rate", metavar="RATE", action=ConfigAction,
help="Maximum download rate (e.g. 500k or 2.5M)",
)
downloader.add_argument(
"--chunk-size",
dest="chunk-size", metavar="SIZE", action=ConfigAction,
help="Size of in-memory data chunks (default: 32k)",
)
downloader.add_argument(
"--sleep",
dest="sleep", metavar="SECONDS", action=ConfigAction,
help=("Number of seconds to wait before each download. "
"This can be either a constant value or a range "
"(e.g. 2.7 or 2.0-3.5)"),
)
downloader.add_argument(
"--sleep-request",
dest="sleep-request", metavar="SECONDS", action=ConfigAction,
help=("Number of seconds to wait between HTTP requests "
"during data extraction"),
)
downloader.add_argument(
"--sleep-extractor",
dest="sleep-extractor", metavar="SECONDS", action=ConfigAction,
help=("Number of seconds to wait before starting data extraction "
"for an input URL"),
)
downloader.add_argument(
"--no-part",
dest="part", nargs=0, action=ConfigConstAction, const=False,
help="Do not use .part files",
)
downloader.add_argument(
"--no-skip",
dest="skip", nargs=0, action=ConfigConstAction, const=False,
help="Do not skip downloads; overwrite existing files",
)
downloader.add_argument(
"--no-mtime",
dest="mtime", nargs=0, action=ConfigConstAction, const=False,
help=("Do not set file modification times according to "
"Last-Modified HTTP response headers")
)
downloader.add_argument(
"--no-download",
dest="download", nargs=0, action=ConfigConstAction, const=False,
help=("Do not download any files")
)
configuration = parser.add_argument_group("Configuration Options")
configuration.add_argument(
"-o", "--option",
dest="options", metavar="KEY=VALUE",
action=ConfigParseAction, default=[],
help=("Additional options. "
"Example: -o browser=firefox") ,
)
configuration.add_argument(
"-c", "--config",
dest="configs_json", metavar="FILE", action="append",
help="Additional configuration files",
)
configuration.add_argument(
"--config-yaml",
dest="configs_yaml", metavar="FILE", action="append",
help="Additional configuration files in YAML format",
)
configuration.add_argument(
"--config-toml",
dest="configs_toml", metavar="FILE", action="append",
help="Additional configuration files in TOML format",
)
configuration.add_argument(
"--config-create",
dest="config", action="store_const", const="init",
help="Create a basic configuration file",
)
configuration.add_argument(
"--config-status",
dest="config", action="store_const", const="status",
help="Show configuration file status",
)
configuration.add_argument(
"--config-open",
dest="config", action="store_const", const="open",
help="Open configuration file in external application",
)
configuration.add_argument(
"--config-ignore",
dest="config_load", action="store_false",
help="Do not read default configuration files",
)
configuration.add_argument(
"--ignore-config",
dest="config_load", action="store_false",
help=argparse.SUPPRESS,
)
authentication = parser.add_argument_group("Authentication Options")
authentication.add_argument(
"-u", "--username",
dest="username", metavar="USER", action=ConfigAction,
help="Username to login with",
)
authentication.add_argument(
"-p", "--password",
dest="password", metavar="PASS", action=ConfigAction,
help="Password belonging to the given username",
)
authentication.add_argument(
"--netrc",
dest="netrc", nargs=0, action=ConfigConstAction, const=True,
help="Enable .netrc authentication data",
)
cookies = parser.add_argument_group("Cookie Options")
cookies.add_argument(
"-C", "--cookies",
dest="cookies", metavar="FILE", action=ConfigAction,
help="File to load additional cookies from",
)
cookies.add_argument(
"--cookies-export",
dest="cookies-update", metavar="FILE", action=ConfigAction,
help="Export session cookies to FILE",
)
cookies.add_argument(
"--cookies-from-browser",
dest="cookies_from_browser",
metavar="BROWSER[/DOMAIN][+KEYRING][:PROFILE][::CONTAINER]",
help=("Name of the browser to load cookies from, with optional "
"domain prefixed with '/', "
"keyring name prefixed with '+', "
"profile prefixed with ':', and "
"container prefixed with '::' "
"('none' for no container (default), 'all' for all containers)"),
)
selection = parser.add_argument_group("Selection Options")
selection.add_argument(
"-A", "--abort",
dest="abort", metavar="N", type=int,
help=("Stop current extractor run "
"after N consecutive file downloads were skipped"),
)
selection.add_argument(
"-T", "--terminate",
dest="terminate", metavar="N", type=int,
help=("Stop current and parent extractor runs "
"after N consecutive file downloads were skipped"),
)
selection.add_argument(
"--filesize-min",
dest="filesize-min", metavar="SIZE", action=ConfigAction,
help="Do not download files smaller than SIZE (e.g. 500k or 2.5M)",
)
selection.add_argument(
"--filesize-max",
dest="filesize-max", metavar="SIZE", action=ConfigAction,
help="Do not download files larger than SIZE (e.g. 500k or 2.5M)",
)
selection.add_argument(
"--download-archive",
dest="archive", metavar="FILE", action=ConfigAction,
help=("Record all downloaded or skipped files in FILE and "
"skip downloading any file already in it"),
)
selection.add_argument(
"--range",
dest="image-range", metavar="RANGE", action=ConfigAction,
help=("Index range(s) specifying which files to download. "
"These can be either a constant value, range, or slice "
"(e.g. '5', '8-20', or '1:24:3')"),
)
selection.add_argument(
"--chapter-range",
dest="chapter-range", metavar="RANGE", action=ConfigAction,
help=("Like '--range', but applies to manga chapters "
"and other delegated URLs"),
)
selection.add_argument(
"--filter",
dest="image-filter", metavar="EXPR", action=ConfigAction,
help=("Python expression controlling which files to download. "
"Files for which the expression evaluates to False are ignored. "
"Available keys are the filename-specific ones listed by '-K'. "
"Example: --filter \"image_width >= 1000 and "
"rating in ('s', 'q')\""),
)
selection.add_argument(
"--chapter-filter",
dest="chapter-filter", metavar="EXPR", action=ConfigAction,
help=("Like '--filter', but applies to manga chapters "
"and other delegated URLs"),
)
infojson = {
"name" : "metadata",
"event" : "init",
"filename": "info.json",
}
postprocessor = parser.add_argument_group("Post-processing Options")
postprocessor.add_argument(
"-P", "--postprocessor",
dest="postprocessors", metavar="NAME", action="append", default=[],
help="Activate the specified post processor",
)
postprocessor.add_argument(
"--no-postprocessors",
dest="postprocess", nargs=0, action=ConfigConstAction, const=False,
help=("Do not run any post processors")
)
postprocessor.add_argument(
"-O", "--postprocessor-option",
dest="options_pp", metavar="KEY=VALUE",
action=PPParseAction, default={},
help="Additional post processor options",
)
postprocessor.add_argument(
"--write-metadata",
dest="postprocessors",
action="append_const", const="metadata",
help="Write metadata to separate JSON files",
)
postprocessor.add_argument(
"--write-info-json",
dest="postprocessors",
action="append_const", const=infojson,
help="Write gallery metadata to a info.json file",
)
postprocessor.add_argument(
"--write-infojson",
dest="postprocessors",
action="append_const", const=infojson,
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--write-tags",
dest="postprocessors",
action="append_const", const={"name": "metadata", "mode": "tags"},
help="Write image tags to separate text files",
)
postprocessor.add_argument(
"--zip",
dest="postprocessors",
action="append_const", const="zip",
help="Store downloaded files in a ZIP archive",
)
postprocessor.add_argument(
"--cbz",
dest="postprocessors",
action="append_const", const={
"name" : "zip",
"extension": "cbz",
},
help="Store downloaded files in a CBZ archive",
)
postprocessor.add_argument(
"--mtime",
dest="postprocessors", metavar="NAME", action=MtimeAction,
help=("Set file modification times according to metadata "
"selected by NAME. Examples: 'date' or 'status[date]'"),
)
postprocessor.add_argument(
"--mtime-from-date",
dest="postprocessors", nargs=0, action=MtimeAction,
const="date|status[date]",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--rename",
dest="postprocessors", metavar="FORMAT", action=RenameAction, const=0,
help=("Rename previously downloaded files from FORMAT "
"to the current filename format"),
)
postprocessor.add_argument(
"--rename-to",
dest="postprocessors", metavar="FORMAT", action=RenameAction, const=1,
help=("Rename previously downloaded files from the current filename "
"format to FORMAT"),
)
postprocessor.add_argument(
"--ugoira",
dest="postprocessors", metavar="FMT", action=UgoiraAction,
help=("Convert Pixiv Ugoira to FMT using FFmpeg. "
"Supported formats are 'webm', 'mp4', 'gif', "
"'vp8', 'vp9', 'vp9-lossless', 'copy'."),
)
postprocessor.add_argument(
"--ugoira-conv",
dest="postprocessors", nargs=0, action=UgoiraAction, const="vp8",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--ugoira-conv-lossless",
dest="postprocessors", nargs=0, action=UgoiraAction,
const="vp9-lossless",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--ugoira-conv-copy",
dest="postprocessors", nargs=0, action=UgoiraAction, const="copy",
help=argparse.SUPPRESS,
)
postprocessor.add_argument(
"--exec",
dest="postprocessors", metavar="CMD",
action=AppendCommandAction, const={"name": "exec"},
help=("Execute CMD for each downloaded file. "
"Supported replacement fields are "
"{} or {_path}, {_directory}, {_filename}. "
"Example: --exec \"convert {} {}.png && rm {}\""),
)
postprocessor.add_argument(
"--exec-after",
dest="postprocessors", metavar="CMD",
action=AppendCommandAction, const={
"name": "exec", "event": "finalize"},
help=("Execute CMD after all files were downloaded. "
"Example: --exec-after \"cd {_directory} "
"&& convert * ../doc.pdf\""),
)
return parser