Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-22 10:42:34 +01:00
Code Issues Releases Wiki Activity

[danbooru] reduce API requests for fetching extended 'metadata'

Browse Source 
Instead of using one additional API request per post object (N+1),
this requires only one request per 200-post batch.
This commit is contained in:
Mike Fährmann 2023-04-02 20:11:52 +02:00
parent 421db26aff
commit 75666cf9c3
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/configuration.rst
View File

@ -1148,7 +1148,7 @@ Description
See `available_includes <https://github.com/danbooru/danbooru/blob/2cf7baaf6c5003c1a174a8f2d53db010cf05dca7/app/models/post.rb#L1842-L1849>`__
for possible field names. ``aibooru`` also supports ``ai_metadata``.
Note: This requires 1 additional HTTP request per post.
Note: This requires 1 additional HTTP request per 200-post batch.
extractor.{Danbooru].threshold

26
gallery_dl/extractor/danbooru.py
View File

@ -26,6 +26,7 @@ class DanbooruExtractor(BaseExtractor):
BaseExtractor.__init__(self, match)
self.ugoira = self.config("ugoira", False)
self.external = self.config("external", False)
self.includes = False
threshold = self.config("threshold")
if isinstance(threshold, int):
@ -54,6 +55,7 @@ class DanbooruExtractor(BaseExtractor):
includes = ",".join(includes)
elif not isinstance(includes, str):
includes = "artist_commentary,children,notes,parent,uploader"
self.includes = includes + ",id"
data = self.metadata()
for post in self.posts():
@ -77,11 +79,6 @@ class DanbooruExtractor(BaseExtractor):
url = post["large_file_url"]
post["extension"] = "webm"
if includes:
meta_url = "{}/posts/{}.json?only={}".format(
self.root, post["id"], includes)
post.update(self.request(meta_url).json())
if url[0] == "/":
url = self.root + url
@ -104,6 +101,19 @@ class DanbooruExtractor(BaseExtractor):
posts = self.request(url, params=params).json()
if "posts" in posts:
posts = posts["posts"]
if self.includes and posts:
if not pages and "only" not in params:
params["page"] = "b{}".format(posts[0]["id"] + 1)
params["only"] = self.includes
data = {
meta["id"]: meta
for meta in self.request(url, params=params).json()
}
for post in posts:
post.update(data[post["id"]])
params["only"] = None
yield from posts
if len(posts) < self.threshold:
@ -255,7 +265,11 @@ class DanbooruPostExtractor(DanbooruExtractor):
def posts(self):
url = "{}/posts/{}.json".format(self.root, self.post_id)
return (self.request(url).json(),)
post = self.request(url).json()
if self.includes:
params = {"only": self.includes}
post.update(self.request(url, params=params).json())
return (post,)
class DanbooruPopularExtractor(DanbooruExtractor):