Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2025-02-01 03:51:42 +01:00
Code Issues Releases Wiki Activity

[common] add 'sleep-429' option (#5160)

Browse Source
This commit is contained in:
Mike Fährmann 2024-04-16 18:41:28 +02:00
parent 923c6f3214
commit 566472f080
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 29 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
docs/configuration.rst
View File

@ -358,13 +358,24 @@ Description
i.e. before starting a new extractor.
extractor.*.sleep-429
---------------------
Type
|Duration|_
Default
``60``
Description
Number of seconds to sleep when receiving a `429 Too Many Requests`
response before `retrying <extractor.*.retries_>`__ the request.
extractor.*.sleep-request
-------------------------
Type
|Duration|_
Default
* ``"0.5-1.5"``
``[Danbooru]``, ``[E621]``, ``[foolfuuka]``, ``itaku``,
``[Danbooru]``, ``[E621]``, ``[foolfuuka]:search``, ``itaku``,
``newgrounds``, ``[philomena]``, ``pixiv:novel``, ``plurk``,
``poipiku`` , ``pornpics``, ``soundgasm``, ``urlgalleries``,
``vk``, ``zerochan``

24
gallery_dl/extractor/common.py
View File

@ -198,7 +198,10 @@ class Extractor():
if b'name="captcha-bypass"' in content:
self.log.warning("Cloudflare CAPTCHA")
break
if code not in retry_codes and code < 500:
if code == 429 and self._interval_429:
pass
elif code not in retry_codes and code < 500:
break
finally:
@ -208,14 +211,18 @@ class Extractor():
if tries > retries:
break
seconds = tries
if self._interval:
seconds = self._interval()
if seconds < tries:
seconds = tries
s = self._interval()
if seconds < s:
seconds = s
if code == 429 and self._interval_429:
s = self._interval_429()
if seconds < s:
seconds = s
self.wait(seconds=seconds, reason="429 Too Many Requests")
else:
seconds = tries
self.sleep(seconds, "retry")
self.sleep(seconds, "retry")
tries += 1
raise exception.HttpError(msg, response)
@ -293,6 +300,9 @@ class Extractor():
self.config("sleep-request", self.request_interval),
self.request_interval_min,
)
self._interval_429 = util.build_duration_func(
self.config("sleep-429", 60),
)
if self._retries < 0:
self._retries = float("inf")