mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-25 20:22:36 +01:00
[deviantart] use proper exponential backoff on API errors
... and use separate API credentials for unit tests.
This commit is contained in:
parent
723cc66bb1
commit
557cb94f81
@ -464,29 +464,32 @@ class DeviantartAPI():
|
||||
def _call(self, endpoint, params=None, expect_error=False):
|
||||
"""Call an API endpoint"""
|
||||
url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
|
||||
tries = 1
|
||||
while True:
|
||||
if self.delay:
|
||||
time.sleep(self.delay)
|
||||
if self.delay > 0:
|
||||
time.sleep(2 ** (self.delay-1))
|
||||
|
||||
self.authenticate()
|
||||
response = self.session.get(
|
||||
url, headers=self.headers, params=params)
|
||||
|
||||
if response.status_code == 200:
|
||||
if self.delay > 2:
|
||||
self.delay -= 1
|
||||
break
|
||||
elif response.status_code == 429:
|
||||
|
||||
else:
|
||||
self.delay += 1
|
||||
self.log.debug("rate limit (delay: %d)", self.delay)
|
||||
|
||||
if response.status_code == 429:
|
||||
msg = "Rate limit reached"
|
||||
else:
|
||||
if expect_error:
|
||||
return None
|
||||
self.delay = 1
|
||||
self.log.debug("http status code %d (%d/3)",
|
||||
response.status_code, tries)
|
||||
tries += 1
|
||||
if tries > 3:
|
||||
raise Exception(response.text)
|
||||
msg = "API responded with {} {}".format(
|
||||
response.status_code, response.reason)
|
||||
self.log.warning(
|
||||
"%s. Using %ds delay.", msg, 2 ** (self.delay-1))
|
||||
self.log.debug(response.text)
|
||||
try:
|
||||
return response.json()
|
||||
except ValueError:
|
||||
|
@ -25,7 +25,7 @@ class LusciousAlbumExtractor(AsynchronousExtractor):
|
||||
(("https://luscious.net/c/hentai_manga/albums/"
|
||||
"okinami-no-koigokoro_277031/view/"), {
|
||||
"url": "7e4984a271a1072ac6483e4228a045895aff86f3",
|
||||
"keyword": "76e099479b180420fd5cf820f00c52fe07fda884",
|
||||
"keyword": "e746a3771f4e14bfff81215232a12417d6351ce6",
|
||||
"content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3",
|
||||
}),
|
||||
("https://luscious.net/albums/virgin-killer-sweater_282582/", {
|
||||
|
@ -36,6 +36,9 @@ class TestExtractors(unittest.TestCase):
|
||||
config.set(("extractor", "password"), name)
|
||||
config.set(("extractor", "nijie", "username"), email)
|
||||
config.set(("extractor", "seiga", "username"), email)
|
||||
config.set(("extractor", "deviantart", "client-id"), "7777")
|
||||
config.set(("extractor", "deviantart", "client-secret"),
|
||||
"ff14994c744d9208e5caeec7aab4a026")
|
||||
config.set(("extractor", "tumblr", "api-key"),
|
||||
"0cXoHfIqVzMQcc3HESZSNsVlulGxEXGDTTZCDrRrjaa0jmuTc6")
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user