Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-24 03:32:33 +01:00
Code Issues Releases Wiki Activity

[deviantart] improve API error handling

Browse Source 
The previous implementation would retry requests with 4xx status codes
in an infinite loop, which is especially a problem when querying
non-existent users or groups. These are now properly handled with a
NotFoundError exception.
This commit is contained in:
Mike Fährmann 2018-04-22 17:43:11 +02:00
parent e54b43be08
commit 80521ae1f6
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
1 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
gallery_dl/extractor/deviantart.py
View File

@ -178,6 +178,9 @@ class DeviantartGalleryExtractor(DeviantartExtractor):
"url": "fa6ecb2c3aa78872f762d43f7809b7f0580debc1",
"keyword": "b29746bac291d8c8e339f0256a2bd7bb3ebe1741",
}),
("http://shimoda8.deviantart.com/gallery/", {
"exception": exception.NotFoundError,
}),
("http://shimoda7.deviantart.com/gallery/?catpath=/", None),
]
@ -306,7 +309,6 @@ class DeviantartCollectionExtractor(DeviantartExtractor):
test = [(("https://pencilshadings.deviantart.com"
"/favourites/70595441/3D-Favorites"), {
"url": "742f92199d5bc6a89cda6ec6133d46c7a523824d",
"keyword": "d258ca05424b3586c4feec940158bca00acdf511",
"options": (("original", False),),
})]
@ -355,7 +357,7 @@ class DeviantartAPI():
self.session = extractor.session
self.log = extractor.log
self.headers = {}
self.delay = 0
self.delay = -1
self.mature = extractor.config("mature", "true")
if not isinstance(self.mature, str):
@ -465,35 +467,33 @@ class DeviantartAPI():
"""Call an API endpoint"""
url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
while True:
if self.delay > 0:
time.sleep(2 ** (self.delay-1))
if self.delay >= 0:
time.sleep(2 ** self.delay)
self.authenticate()
response = self.session.get(
url, headers=self.headers, params=params)
data = response.json()
status = response.status_code
if response.status_code == 200:
if self.delay > 2:
if 200 <= status < 400:
if self.delay > 1:
self.delay -= 1
break
return data
elif expect_error:
return None
elif data.get("error_description") == "User not found.":
raise exception.NotFoundError("user or group")
self.log.debug(response.text)
msg = "API responded with {} {}".format(
status, response.reason)
if 400 <= status < 500 and status != 429:
self.log.error(msg)
return data
else:
if response.status_code == 429:
msg = "Rate limit reached"
else:
if expect_error:
return None
msg = "API responded with {} {}".format(
response.status_code, response.reason)
self.delay += 1
self.log.warning(
"%s. Using %ds delay.", msg, 2 ** (self.delay-1))
self.log.debug(response.text)
try:
return response.json()
except ValueError:
self.log.error("Failed to parse API response")
return {}
self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
def _pagination(self, endpoint, params=None):
while True: