Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-22 10:42:34 +01:00
Code Issues Releases Wiki Activity

[deviantart] handle "Request blocked" errors (#655)

Browse Source 
- add a 2 second wait time between requests to deviantart.com
- catch 403 "Request blocked" errors and wait for 3 minutes until
  retrying
This commit is contained in:
Mike Fährmann 2020-04-02 22:14:02 +02:00
parent c874684f05
commit ff7c0b7eff
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
gallery_dl/extractor/deviantart.py
View File

@ -1066,6 +1066,7 @@ class DeviantartOAuthAPI():
class DeviantartEclipseAPI(): class DeviantartEclipseAPI():
"""Interface to the DeviantArt Eclipse API""" """Interface to the DeviantArt Eclipse API"""
_last_request = 0
def __init__(self, extractor): def __init__(self, extractor):
self.extractor = extractor self.extractor = extractor
@ -1102,6 +1103,10 @@ class DeviantartEclipseAPI():
return self._pagination(endpoint, params) return self._pagination(endpoint, params)
def _call(self, endpoint, params=None): def _call(self, endpoint, params=None):
diff = time.time() - DeviantartEclipseAPI._last_request
if diff < 2.0:
time.sleep(2.0 - diff)
url = "https://www.deviantart.com/_napi/" + endpoint url = "https://www.deviantart.com/_napi/" + endpoint
headers = {"Referer": "https://www.deviantart.com/"} headers = {"Referer": "https://www.deviantart.com/"}
@ -1114,8 +1119,8 @@ class DeviantartEclipseAPI():
raise exception.StopExtraction( raise exception.StopExtraction(
"Your account must use the Eclipse interface.") "Your account must use the Eclipse interface.")
elif code == 403 and b"Request blocked." in response.content: elif code == 403 and b"Request blocked." in response.content:
raise exception.StopExtraction( self.extractor.wait(seconds=180, reason="rate limit reset")
"Requests to deviantart.com blocked due to too much traffic.") return self._call(endpoint, params)
try: try:
return response.json() return response.json()
except Exception: except Exception: