[horne] add support for horne.red (#2700)

2024-11-22 10:42:34 +01:00 · 2022-06-25 14:24:46 +02:00 · 2022-06-25 14:24:46 +02:00 · 241e82e18d
commit 241e82e18d
parent 7af4d2047b
5 changed files with 171 additions and 64 deletions
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@ -541,12 +541,6 @@ Consider all sites to be NSFW unless otherwise known.
    <td>individual Images, User Profiles</td>
    <td><a href="https://github.com/mikf/gallery-dl#cookies">Cookies</a></td>
 </tr>
-<tr>
-    <td>nijie</td>
-    <td>https://nijie.info/</td>
-    <td>Doujin, Favorites, Illustrations, individual Images, Nuita History, User Profiles</td>
-    <td>Required</td>
-</tr>
 <tr>
    <td>Nozomi.la</td>
    <td>https://nozomi.la/</td>
@ -1004,6 +998,22 @@ Consider all sites to be NSFW unless otherwise known.
    <td></td>
 </tr>

+<tr>
+    <td colspan="4"><strong>Nijie Instances</strong></td>
+</tr>
+<tr>
+    <td>nijie</td>
+    <td>https://nijie.info/</td>
+    <td>Doujin, Favorites, Illustrations, individual Images, Nuita History, User Profiles</td>
+    <td>Required</td>
+</tr>
+<tr>
+    <td>horne</td>
+    <td>https://horne.red/</td>
+    <td>Doujin, Favorites, Illustrations, individual Images, Nuitas, User Profiles</td>
+    <td>Required</td>
+</tr>
+
 <tr>
    <td colspan="4"><strong>Philomena Instances</strong></td>
 </tr>
--- a/gallery_dl/extractor/nijie.py
+++ b/gallery_dl/extractor/nijie.py
@ -6,31 +6,31 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.

-"""Extractors for https://nijie.info/"""
+"""Extractors for nijie instances"""

-from .common import Extractor, Message, AsynchronousMixin
+from .common import BaseExtractor, Message, AsynchronousMixin
 from .. import text, exception
 from ..cache import cache


-BASE_PATTERN = r"(?:https?://)?(?:www\.)?nijie\.info"
-
-
-class NijieExtractor(AsynchronousMixin, Extractor):
+class NijieExtractor(AsynchronousMixin, BaseExtractor):
    """Base class for nijie extractors"""
-    category = "nijie"
+    basecategory = "Nijie"
    directory_fmt = ("{category}", "{user_id}")
    filename_fmt = "{image_id}_p{num}.{extension}"
    archive_fmt = "{image_id}_{num}"
-    cookiedomain = "nijie.info"
-    cookienames = ("nemail", "nlogin")
-    root = "https://nijie.info"
-    view_url = "https://nijie.info/view.php?id="
-    popup_url = "https://nijie.info/view_popup.php?id="

    def __init__(self, match):
-        Extractor.__init__(self, match)
-        self.user_id = text.parse_int(match.group(1))
+        self._init_category(match)
+        self.cookiedomain = "." + self.root.rpartition("/")[2]
+        self.cookienames = (self.category + "_tok",)
+
+        if self.category == "horne":
+            self._extract_data = self._extract_data_horne
+
+        BaseExtractor.__init__(self, match)
+
+        self.user_id = text.parse_int(match.group(match.lastindex))
        self.user_name = None
        self.session.headers["Referer"] = self.root + "/"

@ -39,13 +39,21 @@ class NijieExtractor(AsynchronousMixin, Extractor):

        for image_id in self.image_ids():

-            response = self.request(self.view_url + image_id, fatal=False)
+            url = "{}/view.php?id={}".format(self.root, image_id)
+            response = self.request(url, fatal=False)
            if response.status_code >= 400:
                continue
            page = response.text

            data = self._extract_data(page)
            data["image_id"] = text.parse_int(image_id)
+
+            if self.user_name:
+                data["user_id"] = self.user_id
+                data["user_name"] = self.user_name
+            else:
+                data["user_id"] = data["artist_id"]
+                data["user_name"] = data["artist_name"]
            yield Message.Directory, data

            for image in self._extract_images(page):
@ -68,24 +76,41 @@ class NijieExtractor(AsynchronousMixin, Extractor):
            "description": text.unescape(extr(
                '"description": "', '"').replace("&amp;", "&")),
            "date"       : text.parse_datetime(extr(
-                '"datePublished": "', '"') + "+0900",
-                "%a %b %d %H:%M:%S %Y%z"),
-            "artist_id"  : text.parse_int(extr(
-                '"sameAs": "https://nijie.info/members.php?id=', '"')),
+                '"datePublished": "', '"'), "%a %b %d %H:%M:%S %Y", 9),
+            "artist_id"  : text.parse_int(extr('/members.php?id=', '"')),
            "artist_name": keywords[1],
            "tags"       : keywords[2:-1],
        }
-        data["user_id"] = data["artist_id"]
-        data["user_name"] = data["artist_name"]
+        return data
+
+    @staticmethod
+    def _extract_data_horne(page):
+        """Extract image metadata from 'page'"""
+        extr = text.extract_from(page)
+        keywords = text.unescape(extr(
+            'name="keywords" content="', '" />')).split(",")
+        data = {
+            "title"      : keywords[0].strip(),
+            "description": text.unescape(extr(
+                'property="og:description" content="', '"')),
+            "artist_id"  : text.parse_int(extr('members.php?id=', '"')),
+            "artist_name": keywords[1],
+            "tags"       : keywords[2:-1],
+            "date"       : text.parse_datetime(extr(
+                "itemprop='datePublished' content=", "<").rpartition(">")[2],
+                "%Y-%m-%d %H:%M:%S", 9),
+        }
        return data

    @staticmethod
    def _extract_images(page):
        """Extract image URLs from 'page'"""
-        images = text.extract_iter(page, '<a href="./view_popup.php', '</a>')
+        images = text.extract_iter(page, "/view_popup.php", "</a>")
        for num, image in enumerate(images):
-            url = "https:" + text.extract(image, 'src="', '"')[0]
-            url = url.replace("/__rs_l120x120/", "/")
+            src = text.extract(image, 'src="', '"')[0]
+            if not src:
+                continue
+            url = ("https:" + src).replace("/__rs_l120x120/", "/")
            yield text.nameext_from_url(url, {
                "num": num,
                "url": url,
@ -112,7 +137,7 @@ class NijieExtractor(AsynchronousMixin, Extractor):
        data = {"email": username, "password": password, "save": "on"}

        response = self.request(url, method="POST", data=data)
-        if "//nijie.info/login.php" in response.text:
+        if "/login.php" in response.text:
            raise exception.AuthenticationError()
        return self.session.cookies

@ -132,12 +157,27 @@ class NijieExtractor(AsynchronousMixin, Extractor):
            params["p"] += 1


+BASE_PATTERN = NijieExtractor.update({
+    "nijie": {
+        "root": "https://nijie.info",
+        "pattern": r"(?:www\.)?nijie\.info",
+    },
+    "horne": {
+        "root": "https://horne.red",
+        "pattern": r"(?:www\.)?horne\.red",
+    },
+})
+
+
 class NijieUserExtractor(NijieExtractor):
    """Extractor for nijie user profiles"""
    subcategory = "user"
    cookiedomain = None
    pattern = BASE_PATTERN + r"/members\.php\?id=(\d+)"
-    test = ("https://nijie.info/members.php?id=44",)
+    test = (
+        ("https://nijie.info/members.php?id=44"),
+        ("https://horne.red/members.php?id=58000"),
+    )

    def items(self):
        fmt = "{}/{{}}.php?id={}".format(self.root, self.user_id).format
@ -172,6 +212,25 @@ class NijieIllustrationExtractor(NijieExtractor):
                "user_name": "ED",
            },
        }),
+        ("https://horne.red/members_illust.php?id=58000", {
+            "pattern": r"https://pic\.nijie\.net/\d+/horne/\d+/\d+/\d+"
+                       r"/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png",
+            "range": "1-20",
+            "count": 20,
+            "keyword": {
+                "artist_id": 58000,
+                "artist_name": "のえるわ",
+                "date": "type:datetime",
+                "description": str,
+                "image_id": int,
+                "num": int,
+                "tags": list,
+                "title": str,
+                "url": str,
+                "user_id": 58000,
+                "user_name": "のえるわ",
+            },
+        }),
        ("https://nijie.info/members_illust.php?id=43", {
            "exception": exception.NotFoundError,
        }),
@ -182,34 +241,47 @@ class NijieIllustrationExtractor(NijieExtractor):


 class NijieDoujinExtractor(NijieExtractor):
-    """Extractor for doujin entries of a nijie-user"""
+    """Extractor for doujin entries of a nijie user"""
    subcategory = "doujin"
    pattern = BASE_PATTERN + r"/members_dojin\.php\?id=(\d+)"
-    test = ("https://nijie.info/members_dojin.php?id=6782", {
+    test = (
+        ("https://nijie.info/members_dojin.php?id=6782", {
            "count": ">= 18",
            "keyword": {
                "user_id"  : 6782,
                "user_name": "ジョニー＠アビオン村",
            },
-    })
+        }),
+        ("https://horne.red/members_dojin.php?id=58000"),
+    )

    def image_ids(self):
        return self._pagination("members_dojin")


 class NijieFavoriteExtractor(NijieExtractor):
-    """Extractor for all favorites/bookmarks of a nijie-user"""
+    """Extractor for all favorites/bookmarks of a nijie user"""
    subcategory = "favorite"
    directory_fmt = ("{category}", "bookmarks", "{user_id}")
    archive_fmt = "f_{user_id}_{image_id}_{num}"
    pattern = BASE_PATTERN + r"/user_like_illust_view\.php\?id=(\d+)"
-    test = ("https://nijie.info/user_like_illust_view.php?id=44", {
+    test = (
+        ("https://nijie.info/user_like_illust_view.php?id=44", {
            "count": ">= 16",
            "keyword": {
                "user_id"  : 44,
                "user_name": "ED",
            },
-    })
+        }),
+        ("https://horne.red/user_like_illust_view.php?id=58000", {
+            "range": "1-5",
+            "count": 5,
+            "keyword": {
+                "user_id"  : 58000,
+                "user_name": "のえるわ",
+            },
+        }),
+    )

    def image_ids(self):
        return self._pagination("user_like_illust_view")
@ -227,14 +299,17 @@ class NijieNuitaExtractor(NijieExtractor):
    directory_fmt = ("{category}", "nuita", "{user_id}")
    archive_fmt = "n_{user_id}_{image_id}_{num}"
    pattern = BASE_PATTERN + r"/history_nuita\.php\?id=(\d+)"
-    test = ("https://nijie.info/history_nuita.php?id=728995", {
+    test = (
+        ("https://nijie.info/history_nuita.php?id=728995", {
            "range": "1-10",
            "count": 10,
            "keyword": {
                "user_id"  : 728995,
                "user_name": "莚",
            },
-    })
+        }),
+        ("https://horne.red/history_nuita.php?id=58000"),
+    )

    def image_ids(self):
        return self._pagination("history_nuita")
@ -252,7 +327,7 @@ class NijieNuitaExtractor(NijieExtractor):


 class NijieImageExtractor(NijieExtractor):
-    """Extractor for a work/image from nijie.info"""
+    """Extractor for a nijie work/image"""
    subcategory = "image"
    pattern = BASE_PATTERN + r"/view(?:_popup)?\.php\?id=(\d+)"
    test = (
@ -265,11 +340,26 @@ class NijieImageExtractor(NijieExtractor):
            "count": 0,
        }),
        ("https://nijie.info/view_popup.php?id=70720"),
+        ("https://horne.red/view.php?id=8716", {
+            "count": 4,
+            "keyword": {
+                "artist_id": 58000,
+                "artist_name": "のえるわ",
+                "date": "dt:2018-02-04 14:47:24",
+                "description": "ノエル「そんなことしなくても、"
+                               "言ってくれたら咥えるのに・・・♡」",
+                "image_id": 8716,
+                "tags": ["男の娘", "フェラ", "オリキャラ", "うちのこ"],
+                "title": "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」",
+                "user_id": 58000,
+                "user_name": "のえるわ",
+            },
+        }),
    )

    def __init__(self, match):
        NijieExtractor.__init__(self, match)
-        self.image_id = match.group(1)
+        self.image_id = match.group(match.lastindex)

    def image_ids(self):
        return (self.image_id,)
--- a/scripts/supportedsites.py
+++ b/scripts/supportedsites.py
@ -47,6 +47,7 @@ CATEGORY_MAP = {
    "hentaihere"     : "HentaiHere",
    "hentaiimg"      : "Hentai Image",
    "hitomi"         : "Hitomi.la",
+    "horne"          : "horne",
    "idolcomplex"    : "Idol Complex",
    "illusioncardsbooru": "Illusion Game Cards",
    "imagebam"       : "ImageBam",
@ -270,6 +271,7 @@ AUTH_MAP = {
    "fantia"         : _COOKIES,
    "flickr"         : _OAUTH,
    "furaffinity"    : _COOKIES,
+    "horne"          : "Required",
    "idolcomplex"    : "Supported",
    "imgbb"          : "Supported",
    "inkbunny"       : "Supported",
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@ -89,7 +89,7 @@ class TestCookiedict(unittest.TestCase):
        self.assertEqual(sorted(cookies.values()), sorted(self.cdict.values()))

    def test_domain(self):
-        for category in ["exhentai", "idolcomplex", "nijie"]:
+        for category in ["exhentai", "idolcomplex", "nijie", "horne"]:
            extr = _get_extractor(category)
            cookies = extr.session.cookies
            for key in self.cdict:
@ -107,7 +107,8 @@ class TestCookieLogin(unittest.TestCase):
        extr_cookies = {
            "exhentai"   : ("ipb_member_id", "ipb_pass_hash"),
            "idolcomplex": ("login", "pass_hash"),
-            "nijie"      : ("nemail", "nlogin"),
+            "nijie"      : ("nijie_tok",),
+            "horne"      : ("horne_tok",),
        }
        for category, cookienames in extr_cookies.items():
            cookies = {name: "value" for name in cookienames}
@ -199,10 +200,13 @@ class TestCookieUtils(unittest.TestCase):


 def _get_extractor(category):
-    for extr in extractor.extractors():
-        if extr.category == category and hasattr(extr, "_login_impl"):
-            url = next(extr._get_tests())[0]
-            return extr.from_url(url)
+    URLS = {
+        "exhentai"   : "https://exhentai.org/g/1200119/d55c44d3d0/",
+        "idolcomplex": "https://idol.sankakucomplex.com/post/show/1",
+        "nijie"      : "https://nijie.info/view.php?id=1",
+        "horne"      : "https://horne.red/view.php?id=1",
+    }
+    return extractor.find(URLS[category])


 if __name__ == "__main__":
--- a/test/test_results.py
+++ b/test/test_results.py
@ -312,6 +312,7 @@ def setup_test_config():

    config.set(("extractor", "nijie")     , "username", email)
    config.set(("extractor", "seiga")     , "username", email)
+    config.set(("extractor", "horne")     , "username", email2)
    config.set(("extractor", "pinterest") , "username", email2)
    config.set(("extractor", "pinterest") , "username", None)  # login broken