mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-22 02:32:33 +01:00
[rule34vault] update
- use more of BooruExtractor's interface - use BASE_PATTERN - define generic pagination method - parse 'date' metadata - make tags & extended metadata optional
This commit is contained in:
Mike Fährmann
parent
3d0263b3ab
commit
a974c30057
@ -9,122 +9,95 @@
|
||||
from .booru import BooruExtractor
|
||||
from .. import text
|
||||
|
||||
BASE_PATTERN = r"(?:https?://)?rule34vault\.com"
|
||||
|
||||
|
||||
class Rule34vaultExtractor(BooruExtractor):
|
||||
category = "rule34vault"
|
||||
root = "https://rule34vault.com"
|
||||
cdn_root = "https://r34xyz.b-cdn.net"
|
||||
root_cdn = "https://r34xyz.b-cdn.net"
|
||||
filename_fmt = "{category}_{id}.{extension}"
|
||||
per_page = 100
|
||||
|
||||
def _get_file_url(self, post_id, filetype):
|
||||
extension = "jpg" if filetype == "image" else "mp4"
|
||||
return "{}/posts/{}/{}/{}.{}".format(self.cdn_root,
|
||||
post_id // 1000,
|
||||
post_id,
|
||||
post_id,
|
||||
extension)
|
||||
def _file_url(self, post):
|
||||
post_id = post["id"]
|
||||
extension = "jpg" if post["type"] == 0 else "mp4"
|
||||
return "{}/posts/{}/{}/{}.{}".format(
|
||||
self.root_cdn, post_id // 1000, post_id, post_id, extension)
|
||||
|
||||
def _parse_post(self, post_id):
|
||||
def _prepare(self, post):
|
||||
post.pop("files", None)
|
||||
post["date"] = text.parse_datetime(
|
||||
post["created"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
if "tags" in post:
|
||||
post["tags"] = [t["value"] for t in post["tags"]]
|
||||
|
||||
def _tags(self, post, _):
|
||||
if "tags" not in post:
|
||||
post.update(self._fetch_post(post["id"]))
|
||||
|
||||
def _fetch_post(self, post_id):
|
||||
url = "{}/api/v2/post/{}".format(self.root, post_id)
|
||||
data = self.request(url).json()
|
||||
return self.request(url).json()
|
||||
|
||||
tags = " ".join(t["value"].replace(" ", "_") for t in data["tags"])
|
||||
post = {
|
||||
"id": post_id,
|
||||
"tags": tags,
|
||||
"uploader": data["uploader"]["userName"],
|
||||
"score": data.get("likes") or 0,
|
||||
"width": data["width"],
|
||||
"height": data["height"],
|
||||
}
|
||||
post["file_url"] = self._get_file_url(post_id,
|
||||
"image" if data['type'] == 0
|
||||
else "video")
|
||||
def _pagination(self, endpoint, params=None):
|
||||
url = "{}/api{}".format(self.root, endpoint)
|
||||
|
||||
return post
|
||||
if params is None:
|
||||
params = {}
|
||||
params["CountTotal"] = True
|
||||
params["Skip"] = self.page_start * self.per_page
|
||||
params["take"] = self.per_page
|
||||
|
||||
while True:
|
||||
data = self.request(url, method="POST", json=params).json()
|
||||
|
||||
yield from data["items"]
|
||||
|
||||
if params["Skip"] + params["take"] > data["totalCount"]:
|
||||
return
|
||||
if "cursor" in data:
|
||||
params["cursor"] = data["cursor"]
|
||||
params["Skip"] += params["take"]
|
||||
|
||||
|
||||
class Rule34vaultPostExtractor(Rule34vaultExtractor):
|
||||
subcategory = "post"
|
||||
archive_fmt = "{id}"
|
||||
pattern = r"(?:https?://)?rule34vault\.com/post/(\d+)"
|
||||
pattern = BASE_PATTERN + r"/post/(\d+)"
|
||||
example = "https://rule34vault.com/post/399437"
|
||||
|
||||
def __init__(self, match):
|
||||
Rule34vaultExtractor.__init__(self, match)
|
||||
self.post_id = int(match.group(1))
|
||||
|
||||
def posts(self):
|
||||
return (self._parse_post(self.post_id),)
|
||||
return (self._fetch_post(self.groups[0]),)
|
||||
|
||||
|
||||
class Rule34vaultPlaylistExtractor(Rule34vaultExtractor):
|
||||
subcategory = "playlist"
|
||||
directory_fmt = ("{category}", "{playlist_id}")
|
||||
archive_fmt = "t_{playlist_id}_{id}"
|
||||
pattern = r"(?:https?://)?rule34vault\.com/playlists/view/(\d+)"
|
||||
archive_fmt = "p_{playlist_id}_{id}"
|
||||
pattern = BASE_PATTERN + r"/playlists/view/(\d+)"
|
||||
example = "https://rule34vault.com/playlists/view/2"
|
||||
|
||||
def __init__(self, match):
|
||||
Rule34vaultExtractor.__init__(self, match)
|
||||
self.playlist_id = match.group(1)
|
||||
|
||||
def metadata(self):
|
||||
return {"playlist_id": self.playlist_id}
|
||||
return {"playlist_id": self.groups[0]}
|
||||
|
||||
def posts(self):
|
||||
url = "{}/api/v2/post/search/playlist/{}".format(self.root,
|
||||
self.playlist_id)
|
||||
current_page = self.page_start
|
||||
|
||||
while True:
|
||||
payload = {
|
||||
"CountTotal": True,
|
||||
"Skip": current_page * self.per_page,
|
||||
"take": self.per_page,
|
||||
}
|
||||
data = self.request(url, method="POST", json=payload).json()
|
||||
|
||||
for post in data["items"]:
|
||||
yield self._parse_post(post["id"])
|
||||
|
||||
if current_page * self.per_page > data["totalCount"]:
|
||||
return
|
||||
current_page += 1
|
||||
endpoint = "/v2/post/search/playlist/" + self.groups[0]
|
||||
return self._pagination(endpoint)
|
||||
|
||||
|
||||
class Rule34vaultTagExtractor(Rule34vaultExtractor):
|
||||
subcategory = "tag"
|
||||
directory_fmt = ("{category}", "{search_tags}")
|
||||
archive_fmt = "t_{search_tags}_{id}"
|
||||
pattern = r"(?:https?://)?rule34vault\.com/([^/?#]+)$"
|
||||
example = "https://rule34vault.com/not_porn"
|
||||
|
||||
def __init__(self, match):
|
||||
Rule34vaultExtractor.__init__(self, match)
|
||||
self.tags_ = text.unquote(match.group(1)).split("%7C")
|
||||
self.tags = [t.replace("_", " ") for t in self.tags_]
|
||||
pattern = BASE_PATTERN + r"/([^/?#]+)$"
|
||||
example = "https://rule34vault.com/TAG"
|
||||
|
||||
def metadata(self):
|
||||
return {"search_tags": " ".join(self.tags_)}
|
||||
self.tags = text.unquote(self.groups[0]).split("%7C")
|
||||
return {"search_tags": " ".join(self.tags)}
|
||||
|
||||
def posts(self):
|
||||
url = '{}/api/v2/post/search/root'.format(self.root)
|
||||
current_page = self.page_start
|
||||
|
||||
while True:
|
||||
payload = {
|
||||
"CountTotal": True,
|
||||
"Skip": current_page * self.per_page,
|
||||
"take": self.per_page,
|
||||
"includeTags": self.tags,
|
||||
}
|
||||
data = self.request(url, method="POST", json=payload).json()
|
||||
|
||||
for post in data["items"]:
|
||||
yield self._parse_post(post["id"])
|
||||
|
||||
if current_page * self.per_page > data['totalCount']:
|
||||
return
|
||||
current_page += 1
|
||||
endpoint = "/v2/post/search/root"
|
||||
params = {"includeTags": [t.replace("_", " ") for t in self.tags]}
|
||||
return self._pagination(endpoint, params)
|
||||
|
||||
@ -8,31 +8,34 @@ from gallery_dl.extractor import rule34vault
|
||||
|
||||
|
||||
__tests__ = (
|
||||
{
|
||||
"#url" : "https://rule34vault.com/sfw",
|
||||
"#category": ("booru", "rule34vault", "tag"),
|
||||
"#class" : rule34vault.Rule34vaultTagExtractor,
|
||||
"#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
|
||||
"#count" : 10,
|
||||
},
|
||||
{
|
||||
"#url" : "https://rule34vault.com/post/486545",
|
||||
"#category": ("booru", "rule34vault", "post"),
|
||||
"#class" : rule34vault.Rule34vaultPostExtractor,
|
||||
"#pattern" : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg",
|
||||
"#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07",
|
||||
},
|
||||
{
|
||||
"#url" : "https://rule34vault.com/post/382937",
|
||||
"#category": ("booru", "rule34vault", "post"),
|
||||
"#class" : rule34vault.Rule34vaultPostExtractor,
|
||||
"#pattern" : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4",
|
||||
"#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af",
|
||||
},
|
||||
{
|
||||
"#url" : "https://rule34vault.com/playlists/view/20164",
|
||||
"#category": ("booru", "rule34vault", "playlist"),
|
||||
"#class" : rule34vault.Rule34vaultPlaylistExtractor,
|
||||
"#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
|
||||
},
|
||||
{
|
||||
"#url" : "https://rule34vault.com/sfw",
|
||||
"#class": rule34vault.Rule34vaultTagExtractor,
|
||||
"#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
|
||||
"#range" : "1-10",
|
||||
"#count" : 10,
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://rule34vault.com/post/486545",
|
||||
"#class": rule34vault.Rule34vaultPostExtractor,
|
||||
"#pattern" : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg",
|
||||
"#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://rule34vault.com/post/382937",
|
||||
"#comment": "video",
|
||||
"#class" : rule34vault.Rule34vaultPostExtractor,
|
||||
"#pattern" : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4",
|
||||
"#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://rule34vault.com/playlists/view/20164",
|
||||
"#class": rule34vault.Rule34vaultPlaylistExtractor,
|
||||
"#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
|
||||
"#count" : 55,
|
||||
},
|
||||
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user