1
0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-25 20:22:36 +01:00

[deviantart] use proper exponential backoff on API errors

... and use separate API credentials for unit tests.
This commit is contained in:
Mike Fährmann 2018-03-15 15:53:16 +01:00
parent 723cc66bb1
commit 557cb94f81
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
3 changed files with 21 additions and 15 deletions

View File

@ -464,29 +464,32 @@ class DeviantartAPI():
def _call(self, endpoint, params=None, expect_error=False):
"""Call an API endpoint"""
url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
tries = 1
while True:
if self.delay:
time.sleep(self.delay)
if self.delay > 0:
time.sleep(2 ** (self.delay-1))
self.authenticate()
response = self.session.get(
url, headers=self.headers, params=params)
if response.status_code == 200:
if self.delay > 2:
self.delay -= 1
break
elif response.status_code == 429:
else:
self.delay += 1
self.log.debug("rate limit (delay: %d)", self.delay)
if response.status_code == 429:
msg = "Rate limit reached"
else:
if expect_error:
return None
self.delay = 1
self.log.debug("http status code %d (%d/3)",
response.status_code, tries)
tries += 1
if tries > 3:
raise Exception(response.text)
msg = "API responded with {} {}".format(
response.status_code, response.reason)
self.log.warning(
"%s. Using %ds delay.", msg, 2 ** (self.delay-1))
self.log.debug(response.text)
try:
return response.json()
except ValueError:

View File

@ -25,7 +25,7 @@ class LusciousAlbumExtractor(AsynchronousExtractor):
(("https://luscious.net/c/hentai_manga/albums/"
"okinami-no-koigokoro_277031/view/"), {
"url": "7e4984a271a1072ac6483e4228a045895aff86f3",
"keyword": "76e099479b180420fd5cf820f00c52fe07fda884",
"keyword": "e746a3771f4e14bfff81215232a12417d6351ce6",
"content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3",
}),
("https://luscious.net/albums/virgin-killer-sweater_282582/", {

View File

@ -36,6 +36,9 @@ class TestExtractors(unittest.TestCase):
config.set(("extractor", "password"), name)
config.set(("extractor", "nijie", "username"), email)
config.set(("extractor", "seiga", "username"), email)
config.set(("extractor", "deviantart", "client-id"), "7777")
config.set(("extractor", "deviantart", "client-secret"),
"ff14994c744d9208e5caeec7aab4a026")
config.set(("extractor", "tumblr", "api-key"),
"0cXoHfIqVzMQcc3HESZSNsVlulGxEXGDTTZCDrRrjaa0jmuTc6")