From a974c30057b73f1889ee60c38e2e2e8ee47633e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 28 Oct 2024 21:58:54 +0100 Subject: [PATCH] [rule34vault] update - use more of BooruExtractor's interface - use BASE_PATTERN - define generic pagination method - parse 'date' metadata - make tags & extended metadata optional --- gallery_dl/extractor/rule34vault.py | 133 +++++++++++----------------- test/results/rule34vault.py | 57 ++++++------ 2 files changed, 83 insertions(+), 107 deletions(-) diff --git a/gallery_dl/extractor/rule34vault.py b/gallery_dl/extractor/rule34vault.py index df8404ea..de2c0bf6 100644 --- a/gallery_dl/extractor/rule34vault.py +++ b/gallery_dl/extractor/rule34vault.py @@ -9,122 +9,95 @@ from .booru import BooruExtractor from .. import text +BASE_PATTERN = r"(?:https?://)?rule34vault\.com" + class Rule34vaultExtractor(BooruExtractor): category = "rule34vault" root = "https://rule34vault.com" - cdn_root = "https://r34xyz.b-cdn.net" + root_cdn = "https://r34xyz.b-cdn.net" filename_fmt = "{category}_{id}.{extension}" per_page = 100 - def _get_file_url(self, post_id, filetype): - extension = "jpg" if filetype == "image" else "mp4" - return "{}/posts/{}/{}/{}.{}".format(self.cdn_root, - post_id // 1000, - post_id, - post_id, - extension) + def _file_url(self, post): + post_id = post["id"] + extension = "jpg" if post["type"] == 0 else "mp4" + return "{}/posts/{}/{}/{}.{}".format( + self.root_cdn, post_id // 1000, post_id, post_id, extension) - def _parse_post(self, post_id): + def _prepare(self, post): + post.pop("files", None) + post["date"] = text.parse_datetime( + post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + if "tags" in post: + post["tags"] = [t["value"] for t in post["tags"]] + + def _tags(self, post, _): + if "tags" not in post: + post.update(self._fetch_post(post["id"])) + + def _fetch_post(self, post_id): url = "{}/api/v2/post/{}".format(self.root, post_id) - data = self.request(url).json() + return self.request(url).json() - tags = " ".join(t["value"].replace(" ", "_") for t in data["tags"]) - post = { - "id": post_id, - "tags": tags, - "uploader": data["uploader"]["userName"], - "score": data.get("likes") or 0, - "width": data["width"], - "height": data["height"], - } - post["file_url"] = self._get_file_url(post_id, - "image" if data['type'] == 0 - else "video") + def _pagination(self, endpoint, params=None): + url = "{}/api{}".format(self.root, endpoint) - return post + if params is None: + params = {} + params["CountTotal"] = True + params["Skip"] = self.page_start * self.per_page + params["take"] = self.per_page + + while True: + data = self.request(url, method="POST", json=params).json() + + yield from data["items"] + + if params["Skip"] + params["take"] > data["totalCount"]: + return + if "cursor" in data: + params["cursor"] = data["cursor"] + params["Skip"] += params["take"] class Rule34vaultPostExtractor(Rule34vaultExtractor): subcategory = "post" archive_fmt = "{id}" - pattern = r"(?:https?://)?rule34vault\.com/post/(\d+)" + pattern = BASE_PATTERN + r"/post/(\d+)" example = "https://rule34vault.com/post/399437" - def __init__(self, match): - Rule34vaultExtractor.__init__(self, match) - self.post_id = int(match.group(1)) - def posts(self): - return (self._parse_post(self.post_id),) + return (self._fetch_post(self.groups[0]),) class Rule34vaultPlaylistExtractor(Rule34vaultExtractor): subcategory = "playlist" directory_fmt = ("{category}", "{playlist_id}") - archive_fmt = "t_{playlist_id}_{id}" - pattern = r"(?:https?://)?rule34vault\.com/playlists/view/(\d+)" + archive_fmt = "p_{playlist_id}_{id}" + pattern = BASE_PATTERN + r"/playlists/view/(\d+)" example = "https://rule34vault.com/playlists/view/2" - def __init__(self, match): - Rule34vaultExtractor.__init__(self, match) - self.playlist_id = match.group(1) - def metadata(self): - return {"playlist_id": self.playlist_id} + return {"playlist_id": self.groups[0]} def posts(self): - url = "{}/api/v2/post/search/playlist/{}".format(self.root, - self.playlist_id) - current_page = self.page_start - - while True: - payload = { - "CountTotal": True, - "Skip": current_page * self.per_page, - "take": self.per_page, - } - data = self.request(url, method="POST", json=payload).json() - - for post in data["items"]: - yield self._parse_post(post["id"]) - - if current_page * self.per_page > data["totalCount"]: - return - current_page += 1 + endpoint = "/v2/post/search/playlist/" + self.groups[0] + return self._pagination(endpoint) class Rule34vaultTagExtractor(Rule34vaultExtractor): subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" - pattern = r"(?:https?://)?rule34vault\.com/([^/?#]+)$" - example = "https://rule34vault.com/not_porn" - - def __init__(self, match): - Rule34vaultExtractor.__init__(self, match) - self.tags_ = text.unquote(match.group(1)).split("%7C") - self.tags = [t.replace("_", " ") for t in self.tags_] + pattern = BASE_PATTERN + r"/([^/?#]+)$" + example = "https://rule34vault.com/TAG" def metadata(self): - return {"search_tags": " ".join(self.tags_)} + self.tags = text.unquote(self.groups[0]).split("%7C") + return {"search_tags": " ".join(self.tags)} def posts(self): - url = '{}/api/v2/post/search/root'.format(self.root) - current_page = self.page_start - - while True: - payload = { - "CountTotal": True, - "Skip": current_page * self.per_page, - "take": self.per_page, - "includeTags": self.tags, - } - data = self.request(url, method="POST", json=payload).json() - - for post in data["items"]: - yield self._parse_post(post["id"]) - - if current_page * self.per_page > data['totalCount']: - return - current_page += 1 + endpoint = "/v2/post/search/root" + params = {"includeTags": [t.replace("_", " ") for t in self.tags]} + return self._pagination(endpoint, params) diff --git a/test/results/rule34vault.py b/test/results/rule34vault.py index 81980dac..2fe31a1f 100644 --- a/test/results/rule34vault.py +++ b/test/results/rule34vault.py @@ -8,31 +8,34 @@ from gallery_dl.extractor import rule34vault __tests__ = ( - { - "#url" : "https://rule34vault.com/sfw", - "#category": ("booru", "rule34vault", "tag"), - "#class" : rule34vault.Rule34vaultTagExtractor, - "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - "#count" : 10, - }, - { - "#url" : "https://rule34vault.com/post/486545", - "#category": ("booru", "rule34vault", "post"), - "#class" : rule34vault.Rule34vaultPostExtractor, - "#pattern" : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg", - "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07", - }, - { - "#url" : "https://rule34vault.com/post/382937", - "#category": ("booru", "rule34vault", "post"), - "#class" : rule34vault.Rule34vaultPostExtractor, - "#pattern" : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4", - "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", - }, - { - "#url" : "https://rule34vault.com/playlists/view/20164", - "#category": ("booru", "rule34vault", "playlist"), - "#class" : rule34vault.Rule34vaultPlaylistExtractor, - "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - }, +{ + "#url" : "https://rule34vault.com/sfw", + "#class": rule34vault.Rule34vaultTagExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://rule34vault.com/post/486545", + "#class": rule34vault.Rule34vaultPostExtractor, + "#pattern" : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg", + "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07", +}, + +{ + "#url" : "https://rule34vault.com/post/382937", + "#comment": "video", + "#class" : rule34vault.Rule34vaultPostExtractor, + "#pattern" : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4", + "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", +}, + +{ + "#url" : "https://rule34vault.com/playlists/view/20164", + "#class": rule34vault.Rule34vaultPlaylistExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#count" : 55, +}, + )