From 7c12ba1c311955a99bc310f635efb647508283b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 5 Apr 2015 16:23:20 +0200 Subject: [PATCH] move DownloadManager and ExtractorFinder --- gallery_dl/download.py | 110 ++++++++++++++++++++++++++++++ gallery_dl/downloader/__init__.py | 53 -------------- gallery_dl/extractor/__init__.py | 46 ------------- 3 files changed, 110 insertions(+), 99 deletions(-) create mode 100644 gallery_dl/download.py diff --git a/gallery_dl/download.py b/gallery_dl/download.py new file mode 100644 index 00000000..d86e0610 --- /dev/null +++ b/gallery_dl/download.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +""" """ + +import os +import sys +import re +import sqlite3 +import importlib + + +class DownloadManager(): + + def __init__(self, opts, conf): + self.opts = opts + self.conf = conf + self.downloaders = {} + + def add(self, extr): + if self.opts.dest: + dest = self.opts.dest + elif extr.category in self.conf: + dest = self.conf[extr.category].get("destination", "/tmp/") + else: + dest = self.conf["general"].get("destination", "/tmp/") + dest = os.path.join(dest, extr.category, extr.directory) + os.makedirs(dest, exist_ok=True) + + for url, filename in extr: + path = os.path.join(dest, filename) + if os.path.exists(path): + self.print_skip(path) + continue + dl = self.get_downloader(extr, url) + self.print_start(path) + tries = dl.download(url, path) + self.print_success(path, tries) + + def get_downloader(self, extr, url): + end = url.find("://") + proto = url[:end] if end != -1 else "http" + if proto not in self.downloaders: + # import downloader + module = importlib.import_module("."+proto, __package__) + self.downloaders[proto] = module.Downloader + return self.downloaders[proto](extr) + + @staticmethod + def print_start(path): + print(path, end="") + sys.stdout.flush() + + @staticmethod + def print_skip(path): + print("\033[2m", path, "\033[0m", sep="") + + @staticmethod + def print_success(path, tries): + if tries == 0: + print("\r", end="") + print("\r\033[1;32m", path, "\033[0m", sep="") + + +class ExtractorFinder(): + + def __init__(self, config): + self.config = config + self.match_list = list() + if "database" in config["general"]: + path = os.path.expanduser(config["general"]["database"]) + conn = sqlite3.connect(path) + self.load_from_database(conn) + self.load_from_config(config) + + def match(self, url): + for category, regex in self.match_list: + match = regex.match(url) + if match: + module = importlib.import_module("."+category, __package__) + return module.Extractor(match, self.config) + return None + + def load_from_database(self, db): + query = ( + "SELECT regex.re, category.name " + "FROM regex JOIN category " + "ON regex.category_id = category.id" + ) + for row in db.execute(query): + self.add_match(row[1], row[0]) + + def load_from_config(self, conf): + for category in conf: + for key, value in conf[category].items(): + if(key.startswith("regex")): + self.add_match(category, value) + + def add_match(self, category, regex): + try: + # print(category, regex) + self.match_list.append( (category, re.compile(regex)) ) + except: + print("[Warning] [{0}] failed to compile regular expression '{1}'" + .format(category, regex)) diff --git a/gallery_dl/downloader/__init__.py b/gallery_dl/downloader/__init__.py index b32fe10d..8b137891 100644 --- a/gallery_dl/downloader/__init__.py +++ b/gallery_dl/downloader/__init__.py @@ -1,54 +1 @@ -import os -import sys -import importlib -class DownloadManager(): - - def __init__(self, opts, conf): - self.opts = opts - self.conf = conf - self.downloaders = {} - - def add(self, extr): - if self.opts.dest: - dest = self.opts.dest - elif extr.category in self.conf: - dest = self.conf[extr.category].get("destination", "/tmp/") - else: - dest = self.conf["general"].get("destination", "/tmp/") - dest = os.path.join(dest, extr.category, extr.directory) - os.makedirs(dest, exist_ok=True) - - for url, filename in extr: - path = os.path.join(dest, filename) - if os.path.exists(path): - self.print_skip(path) - continue - dl = self.get_downloader(extr, url) - self.print_start(path) - tries = dl.download(url, path) - self.print_success(path, tries) - - def get_downloader(self, extr, url): - end = url.find("://") - proto = url[:end] if end != -1 else "http" - if proto not in self.downloaders: - # import downloader - module = importlib.import_module("."+proto, __package__) - self.downloaders[proto] = module.Downloader - return self.downloaders[proto](extr) - - @staticmethod - def print_start(path): - print(path, end="") - sys.stdout.flush() - - @staticmethod - def print_skip(path): - print("\033[2m", path, "\033[0m", sep="") - - @staticmethod - def print_success(path, tries): - if tries == 0: - print("\r", end="") - print("\r\033[1;32m", path, "\033[0m", sep="") diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 3857084c..8b137891 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -1,47 +1 @@ -import os -import sys -import re -import sqlite3 -import importlib -class ExtractorFinder(): - - def __init__(self, config): - self.config = config - self.match_list = list() - if "database" in config["general"]: - path = os.path.expanduser(config["general"]["database"]) - conn = sqlite3.connect(path) - self.load_from_database(conn) - self.load_from_config(config) - - def match(self, url): - for category, regex in self.match_list: - match = regex.match(url) - if match: - module = importlib.import_module("."+category, __package__) - return module.Extractor(match, self.config) - return None - - def load_from_database(self, db): - query = ( - "SELECT regex.re, category.name " - "FROM regex JOIN category " - "ON regex.category_id = category.id" - ) - for row in db.execute(query): - self.add_match(row[1], row[0]) - - def load_from_config(self, conf): - for category in conf: - for key, value in conf[category].items(): - if(key.startswith("regex")): - self.add_match(category, value) - - def add_match(self, category, regex): - try: - # print(category, regex) - self.match_list.append( (category, re.compile(regex)) ) - except: - print("[Warning] [{0}] failed to compile regular expression '{1}'" - .format(category, regex))