mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-22 02:32:33 +01:00
move DownloadManager and ExtractorFinder
This commit is contained in:
Mike Fährmann
parent
6f75100ef3
commit
7c12ba1c31
110
gallery_dl/download.py
Normal file
110
gallery_dl/download.py
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright 2015 Mike Fährmann
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License version 2 as
|
||||||
|
# published by the Free Software Foundation.
|
||||||
|
|
||||||
|
""" """
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import importlib
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadManager():
|
||||||
|
|
||||||
|
def __init__(self, opts, conf):
|
||||||
|
self.opts = opts
|
||||||
|
self.conf = conf
|
||||||
|
self.downloaders = {}
|
||||||
|
|
||||||
|
def add(self, extr):
|
||||||
|
if self.opts.dest:
|
||||||
|
dest = self.opts.dest
|
||||||
|
elif extr.category in self.conf:
|
||||||
|
dest = self.conf[extr.category].get("destination", "/tmp/")
|
||||||
|
else:
|
||||||
|
dest = self.conf["general"].get("destination", "/tmp/")
|
||||||
|
dest = os.path.join(dest, extr.category, extr.directory)
|
||||||
|
os.makedirs(dest, exist_ok=True)
|
||||||
|
|
||||||
|
for url, filename in extr:
|
||||||
|
path = os.path.join(dest, filename)
|
||||||
|
if os.path.exists(path):
|
||||||
|
self.print_skip(path)
|
||||||
|
continue
|
||||||
|
dl = self.get_downloader(extr, url)
|
||||||
|
self.print_start(path)
|
||||||
|
tries = dl.download(url, path)
|
||||||
|
self.print_success(path, tries)
|
||||||
|
|
||||||
|
def get_downloader(self, extr, url):
|
||||||
|
end = url.find("://")
|
||||||
|
proto = url[:end] if end != -1 else "http"
|
||||||
|
if proto not in self.downloaders:
|
||||||
|
# import downloader
|
||||||
|
module = importlib.import_module("."+proto, __package__)
|
||||||
|
self.downloaders[proto] = module.Downloader
|
||||||
|
return self.downloaders[proto](extr)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def print_start(path):
|
||||||
|
print(path, end="")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def print_skip(path):
|
||||||
|
print("\033[2m", path, "\033[0m", sep="")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def print_success(path, tries):
|
||||||
|
if tries == 0:
|
||||||
|
print("\r", end="")
|
||||||
|
print("\r\033[1;32m", path, "\033[0m", sep="")
|
||||||
|
|
||||||
|
|
||||||
|
class ExtractorFinder():
|
||||||
|
|
||||||
|
def __init__(self, config):
|
||||||
|
self.config = config
|
||||||
|
self.match_list = list()
|
||||||
|
if "database" in config["general"]:
|
||||||
|
path = os.path.expanduser(config["general"]["database"])
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
self.load_from_database(conn)
|
||||||
|
self.load_from_config(config)
|
||||||
|
|
||||||
|
def match(self, url):
|
||||||
|
for category, regex in self.match_list:
|
||||||
|
match = regex.match(url)
|
||||||
|
if match:
|
||||||
|
module = importlib.import_module("."+category, __package__)
|
||||||
|
return module.Extractor(match, self.config)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def load_from_database(self, db):
|
||||||
|
query = (
|
||||||
|
"SELECT regex.re, category.name "
|
||||||
|
"FROM regex JOIN category "
|
||||||
|
"ON regex.category_id = category.id"
|
||||||
|
)
|
||||||
|
for row in db.execute(query):
|
||||||
|
self.add_match(row[1], row[0])
|
||||||
|
|
||||||
|
def load_from_config(self, conf):
|
||||||
|
for category in conf:
|
||||||
|
for key, value in conf[category].items():
|
||||||
|
if(key.startswith("regex")):
|
||||||
|
self.add_match(category, value)
|
||||||
|
|
||||||
|
def add_match(self, category, regex):
|
||||||
|
try:
|
||||||
|
# print(category, regex)
|
||||||
|
self.match_list.append( (category, re.compile(regex)) )
|
||||||
|
except:
|
||||||
|
print("[Warning] [{0}] failed to compile regular expression '{1}'"
|
||||||
|
.format(category, regex))
|
||||||
@ -1,54 +1 @@
|
|||||||
import os
|
|
||||||
import sys
|
|
||||||
import importlib
|
|
||||||
|
|
||||||
class DownloadManager():
|
|
||||||
|
|
||||||
def __init__(self, opts, conf):
|
|
||||||
self.opts = opts
|
|
||||||
self.conf = conf
|
|
||||||
self.downloaders = {}
|
|
||||||
|
|
||||||
def add(self, extr):
|
|
||||||
if self.opts.dest:
|
|
||||||
dest = self.opts.dest
|
|
||||||
elif extr.category in self.conf:
|
|
||||||
dest = self.conf[extr.category].get("destination", "/tmp/")
|
|
||||||
else:
|
|
||||||
dest = self.conf["general"].get("destination", "/tmp/")
|
|
||||||
dest = os.path.join(dest, extr.category, extr.directory)
|
|
||||||
os.makedirs(dest, exist_ok=True)
|
|
||||||
|
|
||||||
for url, filename in extr:
|
|
||||||
path = os.path.join(dest, filename)
|
|
||||||
if os.path.exists(path):
|
|
||||||
self.print_skip(path)
|
|
||||||
continue
|
|
||||||
dl = self.get_downloader(extr, url)
|
|
||||||
self.print_start(path)
|
|
||||||
tries = dl.download(url, path)
|
|
||||||
self.print_success(path, tries)
|
|
||||||
|
|
||||||
def get_downloader(self, extr, url):
|
|
||||||
end = url.find("://")
|
|
||||||
proto = url[:end] if end != -1 else "http"
|
|
||||||
if proto not in self.downloaders:
|
|
||||||
# import downloader
|
|
||||||
module = importlib.import_module("."+proto, __package__)
|
|
||||||
self.downloaders[proto] = module.Downloader
|
|
||||||
return self.downloaders[proto](extr)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def print_start(path):
|
|
||||||
print(path, end="")
|
|
||||||
sys.stdout.flush()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def print_skip(path):
|
|
||||||
print("\033[2m", path, "\033[0m", sep="")
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def print_success(path, tries):
|
|
||||||
if tries == 0:
|
|
||||||
print("\r", end="")
|
|
||||||
print("\r\033[1;32m", path, "\033[0m", sep="")
|
|
||||||
|
|||||||
@ -1,47 +1 @@
|
|||||||
import os
|
|
||||||
import sys
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
import importlib
|
|
||||||
|
|
||||||
class ExtractorFinder():
|
|
||||||
|
|
||||||
def __init__(self, config):
|
|
||||||
self.config = config
|
|
||||||
self.match_list = list()
|
|
||||||
if "database" in config["general"]:
|
|
||||||
path = os.path.expanduser(config["general"]["database"])
|
|
||||||
conn = sqlite3.connect(path)
|
|
||||||
self.load_from_database(conn)
|
|
||||||
self.load_from_config(config)
|
|
||||||
|
|
||||||
def match(self, url):
|
|
||||||
for category, regex in self.match_list:
|
|
||||||
match = regex.match(url)
|
|
||||||
if match:
|
|
||||||
module = importlib.import_module("."+category, __package__)
|
|
||||||
return module.Extractor(match, self.config)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def load_from_database(self, db):
|
|
||||||
query = (
|
|
||||||
"SELECT regex.re, category.name "
|
|
||||||
"FROM regex JOIN category "
|
|
||||||
"ON regex.category_id = category.id"
|
|
||||||
)
|
|
||||||
for row in db.execute(query):
|
|
||||||
self.add_match(row[1], row[0])
|
|
||||||
|
|
||||||
def load_from_config(self, conf):
|
|
||||||
for category in conf:
|
|
||||||
for key, value in conf[category].items():
|
|
||||||
if(key.startswith("regex")):
|
|
||||||
self.add_match(category, value)
|
|
||||||
|
|
||||||
def add_match(self, category, regex):
|
|
||||||
try:
|
|
||||||
# print(category, regex)
|
|
||||||
self.match_list.append( (category, re.compile(regex)) )
|
|
||||||
except:
|
|
||||||
print("[Warning] [{0}] failed to compile regular expression '{1}'"
|
|
||||||
.format(category, regex))
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user