Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-23 03:02:50 +01:00
Code Issues Releases Wiki Activity

reduce wait time growth rate from exponential to linear

Browse Source 
Waiting for 2**N seconds after each error grows too fast.
Simply waiting N seconds seems far more reasonable.
This commit is contained in:
Mike Fährmann 2020-09-06 22:38:25 +02:00
parent bc48514d84
commit f6fd449b59
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
4 changed files with 7 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/configuration.rst
View File

@ -723,9 +723,6 @@ extractor.deviantart.wait-min
Type ``integer``
Default ``0``
Description Minimum wait time in seconds before API requests.
Note: This value will internally be rounded up
to the next power of 2.
=========== =====

2
gallery_dl/downloader/http.py
View File

@ -87,7 +87,7 @@ class HttpDownloader(DownloaderBase):
self.log.warning("%s (%s/%s)", msg, tries, self.retries+1)
if tries > self.retries:
return False
time.sleep(min(2 ** (tries-1), 1800))
time.sleep(tries)
tries += 1
headers = {}

2
gallery_dl/extractor/common.py
View File

@ -123,7 +123,7 @@ class Extractor():
self.log.debug("%s (%s/%s)", msg, tries, retries+1)
if tries > retries:
break
time.sleep(min(2 ** (tries-1), 1800))
time.sleep(tries)
tries += 1
raise exception.HttpError(msg, response)

12
gallery_dl/extractor/deviantart.py
View File

@ -14,7 +14,6 @@ from ..cache import cache, memcache
import collections
import itertools
import mimetypes
import math
import time
import re
@ -837,8 +836,7 @@ class DeviantartOAuthAPI():
self.log = extractor.log
self.headers = {}
delay = extractor.config("wait-min", 0)
self.delay = math.ceil(math.log2(delay)) if delay >= 1 else -1
self.delay = extractor.config("wait-min", 0)
self.delay_min = max(2, self.delay)
self.mature = extractor.config("mature", "true")
@ -993,8 +991,8 @@ class DeviantartOAuthAPI():
"""Call an API endpoint"""
url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
while True:
if self.delay >= 0:
time.sleep(2 ** self.delay)
if self.delay:
time.sleep(self.delay)
self.authenticate(None if public else self.refresh_token_key)
response = self.extractor.request(
@ -1015,9 +1013,9 @@ class DeviantartOAuthAPI():
msg = "API responded with {} {}".format(
status, response.reason)
if status == 429:
if self.delay < 9:
if self.delay < 30:
self.delay += 1
self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
self.log.warning("%s. Using %ds delay.", msg, self.delay)
else:
self.log.error(msg)
return data