2015-04-05 17:15:27 +02:00
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
2016-03-06 21:00:42 +01:00
|
|
|
# Copyright 2014-2016 Mike Fährmann
|
2015-04-05 17:15:27 +02:00
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
|
# published by the Free Software Foundation.
|
|
|
|
|
2014-10-12 21:56:44 +02:00
|
|
|
__author__ = "Mike Fährmann"
|
2016-03-06 21:00:42 +01:00
|
|
|
__copyright__ = "Copyright 2014-2016 Mike Fährmann"
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2015-04-05 17:15:27 +02:00
|
|
|
__license__ = "GPLv2"
|
2016-09-23 08:41:03 +02:00
|
|
|
__version__ = "0.5.2"
|
2014-10-12 21:56:44 +02:00
|
|
|
__maintainer__ = "Mike Fährmann"
|
|
|
|
__email__ = "mike_faehrmann@web.de"
|
|
|
|
|
|
|
|
import os
|
2016-08-06 13:40:49 +02:00
|
|
|
import sys
|
2014-10-12 21:56:44 +02:00
|
|
|
import argparse
|
2016-07-21 13:13:53 +02:00
|
|
|
import json
|
2016-08-05 10:25:31 +02:00
|
|
|
from . import config, extractor, job, exception
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2015-11-14 15:31:07 +01:00
|
|
|
def build_cmdline_parser():
|
2015-04-05 17:15:27 +02:00
|
|
|
parser = argparse.ArgumentParser(
|
2014-10-12 21:56:44 +02:00
|
|
|
description='Download images from various sources')
|
2015-04-05 17:15:27 +02:00
|
|
|
parser.add_argument(
|
2016-09-14 09:51:01 +02:00
|
|
|
"-g", "--get-urls", dest="list_urls", action="store_true",
|
|
|
|
help="print download urls",
|
2015-04-05 17:15:27 +02:00
|
|
|
)
|
|
|
|
parser.add_argument(
|
|
|
|
"-d", "--dest",
|
|
|
|
metavar="DEST",
|
2015-11-14 17:22:56 +01:00
|
|
|
help="destination directory",
|
2015-04-05 17:15:27 +02:00
|
|
|
)
|
2016-07-24 22:18:54 +02:00
|
|
|
parser.add_argument(
|
|
|
|
"-u", "--username",
|
|
|
|
metavar="USER"
|
|
|
|
)
|
|
|
|
parser.add_argument(
|
|
|
|
"-p", "--password",
|
|
|
|
metavar="PASS"
|
|
|
|
)
|
2016-09-14 09:51:01 +02:00
|
|
|
parser.add_argument(
|
|
|
|
"-c", "--config",
|
|
|
|
metavar="CFG", dest="cfgfiles", action="append",
|
|
|
|
help="additional configuration files",
|
|
|
|
)
|
2015-11-10 01:56:31 +01:00
|
|
|
parser.add_argument(
|
|
|
|
"-o", "--option",
|
2015-11-12 02:26:27 +01:00
|
|
|
metavar="OPT", action="append", default=[],
|
2016-07-21 13:13:53 +02:00
|
|
|
help="additional 'key=value' option values",
|
2015-11-10 01:56:31 +01:00
|
|
|
)
|
2015-12-10 02:14:28 +01:00
|
|
|
parser.add_argument(
|
2016-09-14 09:51:01 +02:00
|
|
|
"--list-extractors", dest="list_extractors", action="store_true",
|
|
|
|
help="print a list of extractor classes with description and example URL",
|
2015-11-14 15:11:44 +01:00
|
|
|
)
|
2015-11-13 01:19:01 +01:00
|
|
|
parser.add_argument(
|
2015-12-10 02:14:28 +01:00
|
|
|
"--list-keywords", dest="list_keywords", action="store_true",
|
2015-11-14 17:22:56 +01:00
|
|
|
help="print a list of available keywords for the given URLs",
|
2015-11-13 01:19:01 +01:00
|
|
|
)
|
2016-09-14 09:51:01 +02:00
|
|
|
parser.add_argument(
|
|
|
|
"--list-modules", dest="list_modules", action="store_true",
|
|
|
|
help="print a list of available modules/supported sites",
|
|
|
|
)
|
2016-08-24 14:51:15 +02:00
|
|
|
parser.add_argument(
|
|
|
|
"--version", action="version", version=__version__,
|
|
|
|
help="print program version and exit"
|
|
|
|
)
|
2015-04-05 17:15:27 +02:00
|
|
|
parser.add_argument(
|
|
|
|
"urls",
|
2015-11-14 15:11:44 +01:00
|
|
|
nargs="*", metavar="URL",
|
2015-04-05 17:15:27 +02:00
|
|
|
help="url to download images from"
|
|
|
|
)
|
2015-11-14 15:31:07 +01:00
|
|
|
return parser
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2016-07-21 13:13:53 +02:00
|
|
|
|
|
|
|
def parse_option(opt):
|
|
|
|
try:
|
|
|
|
key, value = opt.split("=", 1)
|
|
|
|
try:
|
|
|
|
value = json.loads(value)
|
2016-09-24 12:05:45 +02:00
|
|
|
except ValueError:
|
2016-07-21 13:13:53 +02:00
|
|
|
pass
|
|
|
|
config.set(key.split("."), value)
|
|
|
|
except ValueError:
|
2016-08-06 13:40:49 +02:00
|
|
|
print("Invalid 'key=value' pair:", opt, file=sys.stderr)
|
2016-07-21 13:13:53 +02:00
|
|
|
|
2014-10-12 21:56:44 +02:00
|
|
|
def main():
|
2015-04-10 17:31:49 +02:00
|
|
|
try:
|
2015-11-14 15:11:44 +01:00
|
|
|
config.load()
|
2015-11-14 15:31:07 +01:00
|
|
|
parser = build_cmdline_parser()
|
|
|
|
args = parser.parse_args()
|
2015-11-14 15:11:44 +01:00
|
|
|
|
2015-11-14 17:22:56 +01:00
|
|
|
if args.cfgfiles:
|
|
|
|
config.load(*args.cfgfiles, strict=True)
|
|
|
|
|
2015-11-14 16:07:10 +01:00
|
|
|
if args.dest:
|
|
|
|
config.set(("base-directory",), args.dest)
|
2016-07-24 22:18:54 +02:00
|
|
|
if args.username:
|
|
|
|
config.set(("username",), args.username)
|
|
|
|
if args.password:
|
|
|
|
config.set(("password",), args.password)
|
2015-11-14 16:07:10 +01:00
|
|
|
|
2015-11-14 15:11:44 +01:00
|
|
|
for opt in args.option:
|
2016-07-21 13:13:53 +02:00
|
|
|
parse_option(opt)
|
2015-11-14 15:11:44 +01:00
|
|
|
|
|
|
|
if args.list_modules:
|
|
|
|
for module_name in extractor.modules:
|
|
|
|
print(module_name)
|
2016-09-14 09:51:01 +02:00
|
|
|
elif args.list_extractors:
|
|
|
|
for extr in extractor.extractors():
|
|
|
|
print(extr.__name__)
|
|
|
|
if extr.__doc__:
|
|
|
|
print(extr.__doc__)
|
|
|
|
if hasattr(extr, "test") and extr.test:
|
|
|
|
print("Example:", extr.test[0][0])
|
|
|
|
print()
|
2015-11-14 15:11:44 +01:00
|
|
|
else:
|
2015-11-14 15:31:07 +01:00
|
|
|
if not args.urls:
|
|
|
|
parser.error("the following arguments are required: URL")
|
2016-07-21 13:13:53 +02:00
|
|
|
|
2015-12-10 02:14:28 +01:00
|
|
|
if args.list_urls:
|
2016-07-14 14:25:56 +02:00
|
|
|
jobtype = job.UrlJob
|
2015-12-10 02:14:28 +01:00
|
|
|
elif args.list_keywords:
|
2016-07-14 14:25:56 +02:00
|
|
|
jobtype = job.KeywordJob
|
2015-12-10 02:14:28 +01:00
|
|
|
else:
|
2016-07-14 14:25:56 +02:00
|
|
|
jobtype = job.DownloadJob
|
2016-07-21 13:13:53 +02:00
|
|
|
|
2015-11-14 15:11:44 +01:00
|
|
|
for url in args.urls:
|
2016-07-14 14:57:42 +02:00
|
|
|
try:
|
|
|
|
jobtype(url).run()
|
|
|
|
except exception.NoExtractorError:
|
2016-08-06 13:40:49 +02:00
|
|
|
print("No suitable extractor found for URL '", url, "'",
|
|
|
|
sep="", file=sys.stderr)
|
2016-07-14 14:57:42 +02:00
|
|
|
except exception.AuthenticationError:
|
|
|
|
print("Authentication failed. Please provide a valid "
|
2016-08-06 13:40:49 +02:00
|
|
|
"username/password pair.", file=sys.stderr)
|
2016-09-24 13:26:19 +02:00
|
|
|
except exception.AuthorizationError:
|
|
|
|
print("You do not have permission to access the resource ",
|
|
|
|
"at '", url, "'", sep="", file=sys.stderr)
|
2016-08-28 16:21:51 +02:00
|
|
|
except exception.NotFoundError as err:
|
|
|
|
res = str(err) or "resource (gallery/image/user)"
|
|
|
|
print("The ", res, " at '", url, "' does not exist",
|
|
|
|
sep="", file=sys.stderr)
|
2016-07-21 13:13:53 +02:00
|
|
|
|
2015-04-10 17:31:49 +02:00
|
|
|
except KeyboardInterrupt:
|
2016-08-06 13:40:49 +02:00
|
|
|
print("\nKeyboardInterrupt", file=sys.stderr)
|
2016-08-05 10:25:31 +02:00
|
|
|
except BrokenPipeError:
|
|
|
|
pass
|
2016-08-28 16:21:51 +02:00
|
|
|
except IOError as err:
|
2016-08-05 10:25:31 +02:00
|
|
|
import errno
|
2016-08-28 16:21:51 +02:00
|
|
|
if err.errno != errno.EPIPE:
|
2016-08-05 10:25:31 +02:00
|
|
|
raise
|