Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-22 18:53:21 +01:00
Code Issues Releases Wiki Activity

embed error messages in StopExtraction exceptions

Browse Source
This commit is contained in:
Mike Fährmann 2019-10-28 16:06:36 +01:00
parent d5e3910270
commit 4409d00141
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
16 changed files with 45 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
gallery_dl/exception.py
View File

@ -100,7 +100,7 @@ class NoExtractorError(GalleryDLException):
class StopExtraction(GalleryDLException):
"""Stop data extraction"""
def __init__(self, message=None):
def __init__(self, message=None, *args):
GalleryDLException.__init__(self)
self.message = message
self.message = message % args if args else message
self.code = 1 if message else 0

19
gallery_dl/extractor/exhentai.py
View File

@ -69,8 +69,7 @@ class ExhentaiExtractor(Extractor):
def login(self):
"""Login and set necessary cookies"""
if self.LIMIT:
self.log.error("Image limit reached!")
raise exception.StopExtraction()
raise exception.StopExtraction("Image limit reached!")
if self._check_cookies(self.cookienames):
return
username, password = self._get_auth_info()
@ -235,9 +234,9 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
url = iurl
data = self._parse_image_info(url)
except IndexError:
self.log.error("Unable to parse image info for '%s'", url)
self.log.debug("Page content:\n%s", page)
raise exception.StopExtraction()
raise exception.StopExtraction(
"Unable to parse image info for '%s'", url)
data["num"] = self.image_num
data["image_token"] = self.key["start"] = extr('var startkey="', '";')
@ -272,9 +271,9 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
url = imgurl
data = self._parse_image_info(url)
except IndexError:
self.log.error("Unable to parse image info for '%s'", url)
self.log.debug("Page content:\n%s", page)
raise exception.StopExtraction()
raise exception.StopExtraction(
"Unable to parse image info for '%s'", url)
data["num"] = request["page"]
data["image_token"] = imgkey
@ -311,12 +310,12 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
self._remaining -= data["cost"]
if self._remaining <= 0:
ExhentaiExtractor.LIMIT = True
url = "{}/s/{}/{}-{}".format(
self.root, data["image_token"], self.gallery_id, data["num"])
self.log.error("Image limit reached! Continue with "
"'%s' as URL after resetting it.", url)
ExhentaiExtractor.LIMIT = True
raise exception.StopExtraction()
raise exception.StopExtraction(
"Image limit reached! Continue with '%s' "
"as URL after resetting it.", url)
def _update_limits(self):
url = "https://e-hentai.org/home.php"

9
gallery_dl/extractor/flickr.py
View File

@ -423,14 +423,15 @@ class FlickrAPI(oauth.OAuth1API):
params["api_key"] = self.api_key
data = self.request(self.API_URL, params=params).json()
if "code" in data:
msg = data.get("message")
self.log.debug("Server response: %s", data)
if data["code"] == 1:
raise exception.NotFoundError(self.extractor.subcategory)
elif data["code"] == 98:
raise exception.AuthenticationError(data.get("message"))
raise exception.AuthenticationError(msg)
elif data["code"] == 99:
raise exception.AuthorizationError()
self.log.error("API call failed: %s", data.get("message"))
raise exception.StopExtraction()
raise exception.AuthorizationError(msg)
raise exception.StopExtraction("API request failed: %s", msg)
return data
def _pagination(self, method, params, key="photos"):

3
gallery_dl/extractor/hbrowse.py
View File

@ -29,8 +29,7 @@ class HbrowseBase():
if not data["manga"] and "<b>Warning</b>" in page:
msg = page.rpartition(">")[2].strip()
self.log.error("Site is not accessible: '%s'", msg)
raise exception.StopExtraction()
raise exception.StopExtraction("Site is not accessible: '%s'", msg)
tags = text.extract(page, 'class="listTable"', '</table>', pos)[0]

3
gallery_dl/extractor/kissmanga.py
View File

@ -33,10 +33,9 @@ class RedirectMixin():
except (EOFError, OSError):
pass
else:
self.log.error(
raise exception.StopExtraction(
"Redirect to \n%s\nVisit this URL in your browser and "
"solve the CAPTCHA to continue", response.url)
raise exception.StopExtraction()
class KissmangaBase(RedirectMixin):

6
gallery_dl/extractor/luscious.py
View File

@ -31,10 +31,10 @@ class LusciousExtractor(Extractor):
)
if response.status_code >= 400:
self.log.error("GraphQL query failed ('%s %s')",
response.status_code, response.reason)
self.log.debug("Server response: %s", response.text)
raise exception.StopExtraction()
raise exception.StopExtraction(
"GraphQL query failed ('%s %s')",
response.status_code, response.reason)
return response.json()["data"]

4
gallery_dl/extractor/oauth.py
View File

@ -296,8 +296,8 @@ class OAuthMastodon(OAuthBase):
data = self.session.post(url, data=data).json()
if "client_id" not in data or "client_secret" not in data:
self.log.error("Failed to register new application: '%s'", data)
raise exception.StopExtraction()
raise exception.StopExtraction(
"Failed to register new application: '%s'", data)
data["client-id"] = data.pop("client_id")
data["client-secret"] = data.pop("client_secret")

5
gallery_dl/extractor/photobucket.py
View File

@ -149,10 +149,9 @@ class PhotobucketImageExtractor(Extractor):
if "message" not in image:
break # success
tries += 1
self.log.debug("'%s'", image["message"])
self.log.debug(image["message"])
else:
self.log.error("%s", image["message"])
raise exception.StopExtraction()
raise exception.StopExtraction(image["message"])
# adjust metadata entries to be at least somewhat similar
# to what the 'album' extractor provides

3
gallery_dl/extractor/pinterest.py
View File

@ -241,9 +241,8 @@ class PinterestAPI():
if response.status_code == 404 or response.history:
resource = self.extractor.subcategory.rpartition("-")[2]
raise exception.NotFoundError(resource)
self.extractor.log.error("API request failed")
self.extractor.log.debug("%s", response.text)
raise exception.StopExtraction()
raise exception.StopExtraction("API request failed")
def _pagination(self, resource, options):
while True:

11
gallery_dl/extractor/pixiv.py
View File

@ -335,11 +335,9 @@ class PixivSearchExtractor(PixivExtractor):
def get_metadata(self, user=None):
query = text.parse_query(self.query)
if "word" in query:
self.word = text.unescape(query["word"])
else:
self.log.error("missing search term")
raise exception.StopExtraction()
if "word" not in query:
raise exception.StopExtraction("Missing search term")
self.word = query["word"]
sort = query.get("order", "date_d")
sort_map = {
@ -504,8 +502,7 @@ class PixivAppAPI():
return response.json()
if response.status_code == 404:
raise exception.NotFoundError()
self.log.error("API request failed: %s", response.text)
raise exception.StopExtraction()
raise exception.StopExtraction("API request failed: %s", response.text)
def _pagination(self, endpoint, params):
while True:

5
gallery_dl/extractor/sankaku.py
View File

@ -201,9 +201,8 @@ class SankakuTagExtractor(SankakuExtractor):
tags = self.tags.split()
if not self.logged_in and len(tags) > 4:
self.log.error("Unauthenticated users cannot use "
"more than 4 tags at once.")
raise exception.StopExtraction()
raise exception.StopExtraction(
"Unauthenticated users cannot use more than 4 tags at once.")
return {"search_tags": " ".join(tags)}
def get_posts(self):

2
gallery_dl/extractor/sexcom.py
View File

@ -52,7 +52,7 @@ class SexcomExtractor(Extractor):
def _parse_pin(self, url):
response = self.request(url, fatal=False)
if response.status_code >= 400:
self.log.warning('Unable to fetch %s ("%s: %s")',
self.log.warning('Unable to fetch %s ("%s %s")',
url, response.status_code, response.reason)
return None
extr = text.extract_from(response.text)

8
gallery_dl/extractor/smugmug.py
View File

@ -259,11 +259,9 @@ class SmugmugAPI(oauth.OAuth1API):
if data["Code"] == 404:
raise exception.NotFoundError()
if data["Code"] == 429:
self.log.error("Rate limit reached")
else:
self.log.error("API request failed")
self.log.debug(data)
raise exception.StopExtraction()
raise exception.StopExtraction("Rate limit reached")
self.log.debug(data)
raise exception.StopExtraction("API request failed")
def _expansion(self, endpoint, expands, params=None):
endpoint = self._extend(endpoint, expands)

7
gallery_dl/extractor/tsumino.py
View File

@ -113,10 +113,9 @@ class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor):
response = self.request(url, headers=headers, fatal=False)
if "/Auth/" in response.url:
self.log.error(
raise exception.StopExtraction(
"Failed to get gallery JSON data. Visit '%s' in a browser "
"and solve the CAPTCHA to continue.", response.url)
raise exception.StopExtraction()
page = response.text
tpl, pos = text.extract(page, 'data-cdn="', '"')
@ -195,8 +194,8 @@ class TsuminoSearchExtractor(TsuminoBase, Extractor):
return self._parse_simple(query)
return self._parse_jsurl(query)
except Exception as exc:
self.log.error("Invalid search query: '%s' (%s)", query, exc)
raise exception.StopExtraction()
raise exception.StopExtraction(
"Invalid search query '%s' (%s)", query, exc)
@staticmethod
def _parse_simple(query):

14
gallery_dl/extractor/tumblr.py
View File

@ -407,26 +407,22 @@ class TumblrAPI(oauth.OAuth1API):
# daily rate limit
if response.headers.get("x-ratelimit-perday-remaining") == "0":
reset = response.headers.get("x-ratelimit-perday-reset")
self.log.error(
raise exception.StopExtraction(
"Daily API rate limit exceeded: aborting; "
"rate limit will reset at %s",
self._to_time(reset),
"rate limit will reset at %s", self._to_time(reset),
)
raise exception.StopExtraction()
# hourly rate limit
reset = response.headers.get("x-ratelimit-perhour-reset")
if reset:
self.log.info(
"Hourly API rate limit exceeded; "
"waiting until %s for rate limit reset",
self._to_time(reset),
"Hourly API rate limit exceeded; waiting until "
"%s for rate limit reset", self._to_time(reset),
)
time.sleep(int(reset) + 1)
return self._call(blog, endpoint, params)
self.log.error(data)
raise exception.StopExtraction()
raise exception.StopExtraction(data)
@staticmethod
def _to_time(reset):

2
gallery_dl/job.py
View File

@ -48,7 +48,7 @@ class Job():
self.dispatch(msg)
except exception.StopExtraction as exc:
if exc.message:
log.error("%s", exc.message)
log.error(exc.message)
return exc.code
except exception.GalleryDLException as exc:
log.error("%s: %s", exc.__class__.__name__, exc)