[rule34vault] Added initial support for rule34vault.com

- Added playlists support for rule34vault
- Added support for posts in rule34vault
- Fixed supported sites with script
- Fixed posts pattern in rule34vault
- Added tests for rule34vault
- Clean
- Fixed lint warnings

docs/supportedsites.md

@@ -757,6 +757,12 @@ Consider all listed sites to potentially be NSFW.
     <td>Galleries, individual Images</td>
     <td></td>
 </tr>
+<tr>
+    <td>R34 Vault</td>
+    <td>https://rule34vault.com/</td>
+    <td>Playlists, Posts, Tag Searches</td>
+    <td></td>
+</tr>
 <tr>
     <td>Read Comic Online</td>
     <td>https://readcomiconline.li/</td>

gallery_dl/extractor/__init__.py

@@ -139,6 +139,7 @@ modules = [
     "reddit",
     "redgifs",
     "rule34us",
+    "rule34vault",
     "sankaku",
     "sankakucomplex",
     "seiga",

gallery_dl/extractor/rule34vault.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://rule34vault.com/"""
+
+from .booru import BooruExtractor
+from .. import text
+
+
+class Rule34vaultExtractor(BooruExtractor):
+    category = "rule34vault"
+    root = "https://rule34vault.com"
+    cdn_root = "https://r34xyz.b-cdn.net"
+    filename_fmt = "{category}_{id}.{extension}"
+    per_page = 100
+
+    def _get_file_url(self, post_id, filetype):
+        extension = "jpg" if filetype == "image" else "mp4"
+        return "{}/posts/{}/{}/{}.{}".format(self.cdn_root,
+                                             post_id // 1000,
+                                             post_id,
+                                             post_id,
+                                             extension)
+
+    def _parse_post(self, post_id):
+        url = "{}/api/v2/post/{}".format(self.root, post_id)
+        data = self.request(url).json()
+
+        tags = " ".join(t["value"].replace(" ", "_") for t in data["tags"])
+        post = {
+            "id": post_id,
+            "tags": tags,
+            "uploader": data["uploader"]["userName"],
+            "score": data.get("likes") or 0,
+            "width": data["width"],
+            "height": data["height"],
+        }
+        post["file_url"] = self._get_file_url(post_id,
+                                              "image" if data['type'] == 0
+                                              else "video")
+
+        return post
+
+
+class Rule34vaultPostExtractor(Rule34vaultExtractor):
+    subcategory = "post"
+    archive_fmt = "{id}"
+    pattern = r"(?:https?://)?rule34vault\.com/post/(\d+)"
+    example = "https://rule34vault.com/post/399437"
+
+    def __init__(self, match):
+        Rule34vaultExtractor.__init__(self, match)
+        self.post_id = int(match.group(1))
+
+    def posts(self):
+        return (self._parse_post(self.post_id),)
+
+
+class Rule34vaultPlaylistExtractor(Rule34vaultExtractor):
+    subcategory = "playlist"
+    directory_fmt = ("{category}", "{playlist_id}")
+    archive_fmt = "t_{playlist_id}_{id}"
+    pattern = r"(?:https?://)?rule34vault\.com/playlists/view/(\d+)"
+    example = "https://rule34vault.com/playlists/view/2"
+
+    def __init__(self, match):
+        Rule34vaultExtractor.__init__(self, match)
+        self.playlist_id = match.group(1)
+
+    def metadata(self):
+        return {"playlist_id": self.playlist_id}
+
+    def posts(self):
+        url = "{}/api/v2/post/search/playlist/{}".format(self.root,
+                                                         self.playlist_id)
+        current_page = self.page_start
+
+        while True:
+            payload = {
+                "CountTotal": True,
+                "Skip": current_page * self.per_page,
+                "take": self.per_page,
+            }
+            data = self.request(url, method="POST", json=payload).json()
+
+            for post in data["items"]:
+                yield self._parse_post(post["id"])
+
+            if current_page * self.per_page > data["totalCount"]:
+                return
+            current_page += 1
+
+
+class Rule34vaultTagExtractor(Rule34vaultExtractor):
+    subcategory = "tag"
+    directory_fmt = ("{category}", "{search_tags}")
+    archive_fmt = "t_{search_tags}_{id}"
+    pattern = r"(?:https?://)?rule34vault\.com/([^/?#]+)$"
+    example = "https://rule34vault.com/not_porn"
+
+    def __init__(self, match):
+        Rule34vaultExtractor.__init__(self, match)
+        self.tags_ = text.unquote(match.group(1)).split("%7C")
+        self.tags = [t.replace("_", " ") for t in self.tags_]
+
+    def metadata(self):
+        return {"search_tags": " ".join(self.tags_)}
+
+    def posts(self):
+        url = '{}/api/v2/post/search/root'.format(self.root)
+        current_page = self.page_start
+
+        while True:
+            payload = {
+                "CountTotal": True,
+                "Skip": current_page * self.per_page,
+                "take": self.per_page,
+                "includeTags": self.tags,
+            }
+            data = self.request(url, method="POST", json=payload).json()
+
+            for post in data["items"]:
+                yield self._parse_post(post["id"])
+
+            if current_page * self.per_page > data['totalCount']:
+                return
+            current_page += 1

scripts/supportedsites.py

@@ -119,6 +119,7 @@ CATEGORY_MAP = {
     "rule34"         : "Rule 34",
     "rule34hentai"   : "Rule34Hentai",
     "rule34us"       : "Rule 34",
+    "rule34vault"    : "R34 Vault",
     "sankaku"        : "Sankaku Channel",
     "sankakucomplex" : "Sankaku Complex",
     "seiga"          : "Niconico Seiga",

test/results/rule34vault.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import rule34vault
+
+
+__tests__ = (
+    {
+        "#url"     : "https://rule34vault.com/sfw",
+        "#category": ("booru", "rule34vault", "tag"),
+        "#class"   : rule34vault.Rule34vaultTagExtractor,
+        "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
+        "#count"   : 10,
+    },
+    {
+        "#url"     : "https://rule34vault.com/post/486545",
+        "#category": ("booru", "rule34vault", "post"),
+        "#class"   : rule34vault.Rule34vaultPostExtractor,
+        "#pattern"     : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg",
+        "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07",
+    },
+    {
+        "#url"     : "https://rule34vault.com/post/382937",
+        "#category": ("booru", "rule34vault", "post"),
+        "#class"   : rule34vault.Rule34vaultPostExtractor,
+        "#pattern"     : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4",
+        "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af",
+    },
+    {
+        "#url"     : "https://rule34vault.com/playlists/view/20164",
+        "#category": ("booru", "rule34vault", "playlist"),
+        "#class"   : rule34vault.Rule34vaultPlaylistExtractor,
+        "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)",
+    },
+)