diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index f726f99c..a34daadf 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -8,10 +8,12 @@ """Common classes and constants used by extractor modules.""" +import time import queue -import threading import requests -# from ..util import safe_request +import threading +import html.parser + class Message(): @@ -95,3 +97,34 @@ class AsyncExtractor(Extractor): import traceback print(traceback.format_exc()) put(None) + + +def safe_request(session, url, method="GET", *args, **kwargs): + tries = 0 + while True: + # try to connect to remote source + try: + r = session.request(method, url, *args, **kwargs) + except requests.exceptions.ConnectionError: + tries += 1 + time.sleep(1) + if tries == 5: + raise + continue + + # reject error-status-codes + if r.status_code != requests.codes.ok: + tries += 1 + time.sleep(1) + if tries == 5: + r.raise_for_status() + continue + + # everything ok -- proceed to download + return r + +def filename_from_url(url): + pos = url.rfind("/") + return url[pos+1:] + +unescape = html.parser.HTMLParser().unescape diff --git a/gallery_dl/util.py b/gallery_dl/util.py deleted file mode 100644 index e5e0cb4f..00000000 --- a/gallery_dl/util.py +++ /dev/null @@ -1,33 +0,0 @@ -import time -import requests -import html.parser - -def safe_request(session, url, method="GET", *args, **kwargs): - tries = 0 - while True: - # try to connect to remote source - try: - r = session.request(method, url, *args, **kwargs) - except requests.exceptions.ConnectionError: - tries += 1 - time.sleep(1) - if tries == 5: - raise - continue - - # reject error-status-codes - if r.status_code != requests.codes.ok: - tries += 1 - time.sleep(1) - if tries == 5: - r.raise_for_status() - continue - - # everything ok -- proceed to download - return r - -def filename_from_url(url): - pos = url.rfind("/") - return url[pos+1:] - -unescape = html.parser.HTMLParser().unescape