From b828473aa3544d666e71eceec98ad6d753daa301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 19 Nov 2018 12:33:34 +0100 Subject: [PATCH] retry HTTP requests for more exception classes --- gallery_dl/extractor/common.py | 11 +++++++---- gallery_dl/job.py | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 5f5e612b..d8f7b476 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -64,9 +64,12 @@ class Extractor(): while True: try: response = self.session.request(method, url, **kwargs) - except (requests.ConnectionError, requests.Timeout) as exc: + except (requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.ContentDecodingError) as exc: msg = exc - except requests.exceptions.RequestException as exc: + except (requests.exceptions.RequestException) as exc: raise exception.HttpError(exc) else: code = response.status_code @@ -79,11 +82,11 @@ class Extractor(): if code < 500 and code != 429: break - self.log.debug("%s (%d/%d)", msg, tries + 1, retries) + tries += 1 + self.log.debug("%s (%d/%d)", msg, tries, retries) if tries >= retries: break time.sleep(2 ** tries) - tries += 1 raise exception.HttpError(msg) diff --git a/gallery_dl/job.py b/gallery_dl/job.py index f5e091f0..8d030d02 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -60,7 +60,10 @@ class Job(): res = str(exc) or "resource (gallery/image/user)" log.error("The %s at '%s' does not exist", res, self.url) except exception.HttpError as exc: - log.error("HTTP request failed: %s", exc) + err = exc.args[0] + if isinstance(err, Exception): + err = "{}: {}".format(err.__class__.__name__, err) + log.error("HTTP request failed: %s", err) except exception.FormatError as exc: err, obj = exc.args log.error("Applying %s format string failed: %s: %s",