[rule34vault] update

- use more of BooruExtractor's interface
- use BASE_PATTERN
- define generic pagination method
- parse 'date' metadata
- make tags & extended metadata optional

gallery_dl/extractor/rule34vault.py

@@ -9,122 +9,95 @@
 from .booru import BooruExtractor
 from .. import text
 
+BASE_PATTERN = r"(?:https?://)?rule34vault\.com"
+
 
 class Rule34vaultExtractor(BooruExtractor):
     category = "rule34vault"
     root = "https://rule34vault.com"
-    cdn_root = "https://r34xyz.b-cdn.net"
+    root_cdn = "https://r34xyz.b-cdn.net"
     filename_fmt = "{category}_{id}.{extension}"
     per_page = 100
 
-    def _get_file_url(self, post_id, filetype):
+    def _file_url(self, post):
-        extension = "jpg" if filetype == "image" else "mp4"
+        post_id = post["id"]
-        return "{}/posts/{}/{}/{}.{}".format(self.cdn_root,
+        extension = "jpg" if post["type"] == 0 else "mp4"
-                                             post_id // 1000,
+        return "{}/posts/{}/{}/{}.{}".format(
-                                             post_id,
+            self.root_cdn, post_id // 1000, post_id, post_id, extension)
-                                             post_id,
-                                             extension)
 
-    def _parse_post(self, post_id):
+    def _prepare(self, post):
+        post.pop("files", None)
+        post["date"] = text.parse_datetime(
+            post["created"], "%Y-%m-%dT%H:%M:%S.%fZ")
+        if "tags" in post:
+            post["tags"] = [t["value"] for t in post["tags"]]
+
+    def _tags(self, post, _):
+        if "tags" not in post:
+            post.update(self._fetch_post(post["id"]))
+
+    def _fetch_post(self, post_id):
         url = "{}/api/v2/post/{}".format(self.root, post_id)
-        data = self.request(url).json()
+        return self.request(url).json()
 
-        tags = " ".join(t["value"].replace(" ", "_") for t in data["tags"])
+    def _pagination(self, endpoint, params=None):
-        post = {
+        url = "{}/api{}".format(self.root, endpoint)
-            "id": post_id,
-            "tags": tags,
-            "uploader": data["uploader"]["userName"],
-            "score": data.get("likes") or 0,
-            "width": data["width"],
-            "height": data["height"],
-        }
-        post["file_url"] = self._get_file_url(post_id,
-                                              "image" if data['type'] == 0
-                                              else "video")
 
-        return post
+        if params is None:
+            params = {}
+        params["CountTotal"] = True
+        params["Skip"] = self.page_start * self.per_page
+        params["take"] = self.per_page
+
+        while True:
+            data = self.request(url, method="POST", json=params).json()
+
+            yield from data["items"]
+
+            if params["Skip"] + params["take"] > data["totalCount"]:
+                return
+            if "cursor" in data:
+                params["cursor"] = data["cursor"]
+            params["Skip"] += params["take"]
 
 
 class Rule34vaultPostExtractor(Rule34vaultExtractor):
     subcategory = "post"
     archive_fmt = "{id}"
-    pattern = r"(?:https?://)?rule34vault\.com/post/(\d+)"
+    pattern = BASE_PATTERN + r"/post/(\d+)"
     example = "https://rule34vault.com/post/399437"
 
-    def __init__(self, match):
-        Rule34vaultExtractor.__init__(self, match)
-        self.post_id = int(match.group(1))
-
     def posts(self):
-        return (self._parse_post(self.post_id),)
+        return (self._fetch_post(self.groups[0]),)
 
 
 class Rule34vaultPlaylistExtractor(Rule34vaultExtractor):
     subcategory = "playlist"
     directory_fmt = ("{category}", "{playlist_id}")
-    archive_fmt = "t_{playlist_id}_{id}"
+    archive_fmt = "p_{playlist_id}_{id}"
-    pattern = r"(?:https?://)?rule34vault\.com/playlists/view/(\d+)"
+    pattern = BASE_PATTERN + r"/playlists/view/(\d+)"
     example = "https://rule34vault.com/playlists/view/2"
 
-    def __init__(self, match):
-        Rule34vaultExtractor.__init__(self, match)
-        self.playlist_id = match.group(1)
-
     def metadata(self):
-        return {"playlist_id": self.playlist_id}
+        return {"playlist_id": self.groups[0]}
 
     def posts(self):
-        url = "{}/api/v2/post/search/playlist/{}".format(self.root,
+        endpoint = "/v2/post/search/playlist/" + self.groups[0]
-                                                         self.playlist_id)
+        return self._pagination(endpoint)
-        current_page = self.page_start
-
-        while True:
-            payload = {
-                "CountTotal": True,
-                "Skip": current_page * self.per_page,
-                "take": self.per_page,
-            }
-            data = self.request(url, method="POST", json=payload).json()
-
-            for post in data["items"]:
-                yield self._parse_post(post["id"])
-
-            if current_page * self.per_page > data["totalCount"]:
-                return
-            current_page += 1
 
 
 class Rule34vaultTagExtractor(Rule34vaultExtractor):
     subcategory = "tag"
     directory_fmt = ("{category}", "{search_tags}")
     archive_fmt = "t_{search_tags}_{id}"
-    pattern = r"(?:https?://)?rule34vault\.com/([^/?#]+)$"
+    pattern = BASE_PATTERN + r"/([^/?#]+)$"
-    example = "https://rule34vault.com/not_porn"
+    example = "https://rule34vault.com/TAG"
-
-    def __init__(self, match):
-        Rule34vaultExtractor.__init__(self, match)
-        self.tags_ = text.unquote(match.group(1)).split("%7C")
-        self.tags = [t.replace("_", " ") for t in self.tags_]
 
     def metadata(self):
-        return {"search_tags": " ".join(self.tags_)}
+        self.tags = text.unquote(self.groups[0]).split("%7C")
+        return {"search_tags": " ".join(self.tags)}
 
     def posts(self):
-        url = '{}/api/v2/post/search/root'.format(self.root)
+        endpoint = "/v2/post/search/root"
-        current_page = self.page_start
+        params = {"includeTags": [t.replace("_", " ") for t in self.tags]}
-
+        return self._pagination(endpoint, params)
-        while True:
-            payload = {
-                "CountTotal": True,
-                "Skip": current_page * self.per_page,
-                "take": self.per_page,
-                "includeTags": self.tags,
-            }
-            data = self.request(url, method="POST", json=payload).json()
-
-            for post in data["items"]:
-                yield self._parse_post(post["id"])
-
-            if current_page * self.per_page > data['totalCount']:
-                return
-            current_page += 1

test/results/rule34vault.py

@@ -8,31 +8,34 @@ from gallery_dl.extractor import rule34vault
 
 
 __tests__ = (
-    {
+{
-        "#url"     : "https://rule34vault.com/sfw",
+    "#url"  : "https://rule34vault.com/sfw",
-        "#category": ("booru", "rule34vault", "tag"),
+    "#class": rule34vault.Rule34vaultTagExtractor,
-        "#class"   : rule34vault.Rule34vaultTagExtractor,
+    "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
-        "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
+    "#range"  : "1-10",
-        "#count"   : 10,
+    "#count"  : 10,
-    },
+},
-    {
+
-        "#url"     : "https://rule34vault.com/post/486545",
+{
-        "#category": ("booru", "rule34vault", "post"),
+    "#url"  : "https://rule34vault.com/post/486545",
-        "#class"   : rule34vault.Rule34vaultPostExtractor,
+    "#class": rule34vault.Rule34vaultPostExtractor,
-        "#pattern"     : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg",
+    "#pattern"     : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg",
-        "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07",
+    "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07",
-    },
+},
-    {
+
-        "#url"     : "https://rule34vault.com/post/382937",
+{
-        "#category": ("booru", "rule34vault", "post"),
+    "#url"    : "https://rule34vault.com/post/382937",
-        "#class"   : rule34vault.Rule34vaultPostExtractor,
+    "#comment": "video",
-        "#pattern"     : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4",
+    "#class"  : rule34vault.Rule34vaultPostExtractor,
-        "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af",
+    "#pattern"     : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4",
-    },
+    "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af",
-    {
+},
-        "#url"     : "https://rule34vault.com/playlists/view/20164",
+
-        "#category": ("booru", "rule34vault", "playlist"),
+{
-        "#class"   : rule34vault.Rule34vaultPlaylistExtractor,
+    "#url"  : "https://rule34vault.com/playlists/view/20164",
-        "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
+    "#class": rule34vault.Rule34vaultPlaylistExtractor,
-    },
+    "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
+    "#count"  : 55,
+},
+
 )