[pixnet] add extractors (closes #177)

for:
- users/blogs: http://albertayu773.pixnet.net/
- folders: https://albertayu773.pixnet.net/album/folder/1405768
- sets   : https://albertayu773.pixnet.net/album/set/15078995
- photos : https://albertayu773.pixnet.net/album/photo/159443828

docs/supportedsites.rst

@@ -73,6 +73,7 @@ Photobucket          https://photobucket.com/            Albums, individual Imag
 Piczel               https://piczel.tv/                  Images from Users, Folders, individual Images
 Pinterest            https://www.pinterest.com/          Boards, Pins, pin.it Links, related Pins
 Pixiv                https://www.pixiv.net/              |pixiv-C|                                          Required
+Pixnet               https://www.pixnet.net/             |pixnet-C|
 Pornreactor          http://pornreactor.cc/              |pornreactor-C|
 PowerManga           https://read.powermanga.org/        Chapters, Manga
 Pururin              https://pururin.io/                 Galleries
@@ -118,6 +119,7 @@ Turboimagehost       https://www.turboimagehost.com/     individual Images
 .. |joyreactor-C| replace:: Images from Users, Posts, Search Results, Tag-Searches
 .. |nijie-C| replace:: Images from Users, Doujin, Favorites, individual Images
 .. |pixiv-C| replace:: Images from Users, Favorites, Follows, pixiv.me Links, Rankings, Search Results, Individual Images
+.. |pixnet-C| replace:: Images from Users, Folders, individual Images, Sets
 .. |pornreactor-C| replace:: Images from Users, Posts, Search Results, Tag-Searches
 .. |smugmug-C| replace:: Albums, individual Images, Images from Users and Folders
 .. |yuki-S| replace:: yuki.la 4chan archive

gallery_dl/extractor/__init__.py

@@ -67,6 +67,7 @@ modules = [
     "piczel",
     "pinterest",
     "pixiv",
+    "pixnet",
     "pururin",
     "reactor",
     "readcomiconline",

gallery_dl/extractor/pixnet.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2016-2019 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://www.pixnet.net/"""
+
+from .common import Extractor, Message
+from .. import text
+
+
+BASE_PATTERN = r"(?:https?://)?(?!www\.)([^.]+)\.pixnet.net"
+
+
+class PixnetExtractor(Extractor):
+    """Base class for pixnet extractors"""
+    category = "pixnet"
+    filename_fmt = "{num:>03}_{id}.{extension}"
+    archive_fmt = "{id}"
+    url_fmt = ""
+
+    def __init__(self, match):
+        Extractor.__init__(self, match)
+        self.blog, self.item_id = match.groups()
+        self.root = "https://{}.pixnet.net".format(self.blog)
+
+    def items(self):
+        url = self.url_fmt.format(self.root, self.item_id)
+        page = self.request(url, encoding="utf-8").text
+        user = text.extract(page, '<meta name="author" content="', '";')[0]
+        data = {
+            "blog": self.blog,
+            "user": user.rpartition(" (")[0],
+        }
+
+        for info in self._pagination(page):
+            url, pos = text.extract(info, ' href="', '"')
+            alt, pos = text.extract(info, ' alt="', '"', pos)
+            item = {
+                "id"        : text.parse_int(url.rpartition("/")[2]),
+                "title"     : text.unescape(alt),
+                "_extractor": (PixnetFolderExtractor if "/folder/" in url else
+                               PixnetSetExtractor),
+            }
+            item.update(data)
+            yield Message.Queue, url, item
+
+    def _pagination(self, page):
+        while True:
+            yield from text.extract_iter(page, '<li id="', '</li>')
+
+            pnext = text.extract(page, 'class="nextBtn"', '>')[0]
+            if "href" not in pnext:
+                return
+            url = self.root + text.extract(pnext, 'href="', '"')[0]
+            page = self.request(url, encoding="utf-8").text
+
+
+class PixnetImageExtractor(PixnetExtractor):
+    """Extractor for a single photo from pixnet.net"""
+    subcategory = "image"
+    filename_fmt = "{id}.{extension}"
+    directory_fmt = ("{category}", "{blog}")
+    pattern = BASE_PATTERN + r"/album/photo/(\d+)"
+    test = ("https://albertayu773.pixnet.net/album/photo/159443828", {
+        "url": "156564c422138914c9fa5b42191677b45c414af4",
+        "keyword": "19971bcd056dfef5593f4328a723a9602be0f087",
+        "content": "0e097bdf49e76dd9b9d57a016b08b16fa6a33280",
+    })
+
+    def items(self):
+        url = "https://api.pixnet.cc/oembed"
+        params = {
+            "url": "https://{}.pixnet.net/album/photo/{}".format(
+                self.blog, self.item_id),
+            "format": "json",
+        }
+
+        data = self.request(url, params=params).json()
+        data["id"] = text.parse_int(
+            data["url"].rpartition("/")[2].partition("-")[0])
+        data["filename"], _, data["extension"] = data["title"].rpartition(".")
+        data["blog"] = self.blog
+        data["user"] = data.pop("author_name")
+
+        yield Message.Version, 1
+        yield Message.Directory, data
+        yield Message.Url, data["url"], data
+
+
+class PixnetSetExtractor(PixnetExtractor):
+    """Extractor for images from a pixnet set"""
+    subcategory = "set"
+    url_fmt = "{}/album/set/{}"
+    directory_fmt = ("{category}", "{blog}",
+                     "{folder_id} {folder_title}", "{set_id} {set_title}")
+    pattern = BASE_PATTERN + r"/album/set/(\d+)"
+    test = (
+        ("https://albertayu773.pixnet.net/album/set/15078995", {
+            "url": "6535712801af47af51110542f4938a7cef44557f",
+            "keyword": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925",
+        }),
+        ("https://anrine910070.pixnet.net/album/set/5917493", {
+            "url": "b3eb6431aea0bcf5003432a4a0f3a3232084fc13",
+            "keyword": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6",
+        }),
+    )
+
+    def items(self):
+        url = self.url_fmt.format(self.root, self.item_id)
+        page = self.request(url, encoding="utf-8").text
+        data = self.metadata(page)
+
+        yield Message.Version, 1
+        yield Message.Directory, data
+        for num, info in enumerate(self._pagination(page), 1):
+            url, pos = text.extract(info, ' href="', '"')
+            src, pos = text.extract(info, ' src="', '"', pos)
+            alt, pos = text.extract(info, ' alt="', '"', pos)
+
+            photo = {
+                "id": text.parse_int(url.rpartition("/")[2].partition("#")[0]),
+                "url": src.replace("_s.", "."),
+                "num": num,
+                "filename": alt,
+                "extension": src.rpartition(".")[2],
+            }
+            photo.update(data)
+            yield Message.Url, photo["url"], photo
+
+    def metadata(self, page):
+        user , pos = text.extract(page, '<meta name="author" content="', '";')
+        _    , pos = text.extract(page, 'id="breadcrumb"', '', pos)
+        fid  , pos = text.extract(page, '/folder/', '"', pos)
+        fname, pos = text.extract(page, '>', '<', pos)
+        sid  , pos = text.extract(page, '/set/', '"', pos)
+        sname, pos = text.extract(page, '>', '<', pos)
+        return {
+            "blog": self.blog,
+            "user": user.rpartition(" (")[0],
+            "folder_id"   : text.parse_int(fid, ""),
+            "folder_title": text.unescape(fname).strip(),
+            "set_id"      : text.parse_int(sid),
+            "set_title"   : text.unescape(sname),
+        }
+
+
+class PixnetFolderExtractor(PixnetExtractor):
+    """Extractor for all sets in a pixnet folder"""
+    subcategory = "folder"
+    url_fmt = "{}/album/folder/{}"
+    pattern = BASE_PATTERN + r"/album/folder/(\d+)"
+    test = ("https://albertayu773.pixnet.net/album/folder/1405768", {
+        "pattern": PixnetSetExtractor.pattern,
+        "count": ">= 15",
+    })
+
+
+class PixnetUserExtractor(PixnetExtractor):
+    """Extractor for all sets and folders of a pixnet user"""
+    subcategory = "user"
+    url_fmt = "{}{}/album/list"
+    pattern = BASE_PATTERN + r"()(?:/blog|/album(?:/list)?)?/?(?:$|[?&#])"
+    test = (
+        ("https://albertayu773.pixnet.net/"),
+        ("https://albertayu773.pixnet.net/blog"),
+        ("https://albertayu773.pixnet.net/album"),
+        ("https://albertayu773.pixnet.net/album/list", {
+            "pattern": PixnetFolderExtractor.pattern,
+            "count": ">= 30",
+        }),
+        ("https://anrine910070.pixnet.net/album/list", {
+            "pattern": PixnetSetExtractor.pattern,
+            "count": ">= 14",
+        }),
+    )