[wikiart] add extractors (#179)

for
- artists:          https://www.wikiart.org/en/thomas-cole
- artist-listings:  https://www.wikiart.org/en/artists-by-century/12
- artwork-listings: https://www.wikiart.org/en/paintings-by-media/grisaille

docs/supportedsites.rst

@@ -93,6 +93,7 @@ Twitter              https://twitter.com/                Media Timelines, Timeli
 Wallhaven            https://alpha.wallhaven.cc/         individual Images, Search Results                  Optional
 Warosu               https://warosu.org/                 Threads
 Weibo                https://www.weibo.com/              Images from Users, Images from Statuses
+WikiArt.org          https://www.wikiart.org/            Artists, Artworks
 World Three          http://www.slide.world-three.org/   Chapters, Manga
 XVideos              https://www.xvideos.com/            Images from Users, Galleries
 Yandere              https://yande.re/                   Pools, Popular Images, Posts, Tag-Searches

gallery_dl/extractor/__init__.py

@@ -83,6 +83,7 @@ modules = [
     "wallhaven",
     "warosu",
     "weibo",
+    "wikiart",
     "yandere",
     "xvideos",
     "yuki",

gallery_dl/extractor/wikiart.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://www.wikiart.org/"""
+
+from .common import Extractor, Message
+from .. import text
+
+BASE_PATTERN = r"(?:https://)?(?:www\.)?wikiart\.org/([a-z]+)"
+
+
+class WikiartExtractor(Extractor):
+    """Base class for wikiart extractors"""
+    category = "wikiart"
+    filename_fmt = "{id}_{title}.{extension}"
+    archive_fmt = "{id}"
+    root = "https://www.wikiart.org"
+
+    def __init__(self, match):
+        Extractor.__init__(self, match)
+        self.lang = match.group(1)
+
+    def items(self):
+        data = self.metadata()
+        yield Message.Version, 1
+        yield Message.Directory, data
+        for painting in self.paintings():
+            url = painting["image"]
+            painting.update(data)
+            yield Message.Url, url, text.nameext_from_url(url, painting)
+
+    def metadata(self):
+        """ """
+
+    def paintings(self):
+        """ """
+
+    def _pagination(self, url, key="Paintings", extra_params=None):
+        headers = {
+            "X-Requested-With": "XMLHttpRequest",
+            "Referer": url,
+        }
+        params = {
+            "json": "2",
+            "layout": "new",
+            "page": 1,
+            "resultType": "masonry",
+        }
+        if extra_params:
+            params.update(extra_params)
+
+        while True:
+            data = self.request(url, headers=headers, params=params).json()
+            items = data.get(key)
+            if not items:
+                return
+            yield from items
+            params["page"] += 1
+
+
+class WikiartArtistExtractor(WikiartExtractor):
+    """Extractor for an artist's paintings on wikiart.org"""
+    subcategory = "artist"
+    directory_fmt = ("{category}", "{artist[artistName]}")
+    pattern = BASE_PATTERN + r"/(?!\w+-by-)([\w-]+)"
+    test = ("https://www.wikiart.org/en/thomas-cole", {
+        "url": "f1eee8158f5b8b7380382ab730a8f53884715c8b",
+        "keyword": "b62678394ce645815963883d5c9642255307225f",
+    })
+
+    def __init__(self, match):
+        WikiartExtractor.__init__(self, match)
+        self.artist = match.group(2)
+
+    def metadata(self):
+        url = "{}/{}/{}?json=2".format(self.root, self.lang, self.artist)
+        return {"artist": self.request(url).json()}
+
+    def paintings(self):
+        """ """
+        url = "{}/{}/{}/mode/all-paintings".format(
+            self.root, self.lang, self.artist)
+        return self._pagination(url)
+
+
+class WikiartArtworksExtractor(WikiartExtractor):
+    """Extractor for artwork collections on wikiart.org"""
+    subcategory = "artworks"
+    directory_fmt = ("{category}", "Artworks by {group!c}", "{type}")
+    pattern = BASE_PATTERN + r"/paintings-by-([\w-]+)/([\w-]+)"
+    test = ("https://www.wikiart.org/en/paintings-by-media/grisaille", {
+        "url": "f92d55669fa949491c26a5437527adb14b35b8cc",
+    })
+
+    def __init__(self, match):
+        WikiartExtractor.__init__(self, match)
+        self.group = match.group(2)
+        self.type = match.group(3)
+
+    def metadata(self):
+        return {"group": self.group, "type": self.type}
+
+    def paintings(self):
+        url = "{}/{}/paintings-by-{}/{}".format(
+            self.root, self.lang, self.group, self.type)
+        return self._pagination(url)
+
+
+class WikiartArtistsExtractor(WikiartExtractor):
+    """Extractor for artist collections on wikiart.org"""
+    subcategory = "artists"
+    pattern = (BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)")
+    test = ("https://www.wikiart.org/en/artists-by-century/12", {
+        "pattern": WikiartArtistExtractor.pattern,
+        "count": 7,
+    })
+
+    def __init__(self, match):
+        WikiartExtractor.__init__(self, match)
+        self.group = match.group(2)
+        self.type = match.group(3)
+
+    def items(self):
+        url = "{}/{}/App/Search/Artists-by-{}".format(
+            self.root, self.lang, self.group)
+        params = {"json": "3", "searchterm": self.type}
+
+        for artist in self._pagination(url, "Artists", params):
+            artist["_extractor"] = WikiartArtistExtractor
+            yield Message.Queue, self.root + artist["artistUrl"], artist

scripts/supportedsites.py

@@ -63,6 +63,7 @@ CATEGORY_MAP = {
     "slideshare"     : "SlideShare",
     "smugmug"        : "SmugMug",
     "thebarchive"    : "The /b/ Archive",
+    "wikiart"        : "WikiArt.org",
     "worldthree"     : "World Three",
     "xvideos"        : "XVideos",
     "yuki"           : "yuki.la 4chan archive",
@@ -70,6 +71,7 @@ CATEGORY_MAP = {
 
 SUBCATEGORY_MAP = {
     "artwork": "Artwork Listings",
+    "artists": "",
     "doujin" : "Doujin",
     "gallery": "Galleries",
     "image"  : "individual Images",

test/test_results.py

@@ -25,8 +25,9 @@ TRAVIS_SKIP = {
 
 # temporary issues, etc.
 BROKEN = {
+    "komikcast",
+    "mangahere",
     "mangapark",
-    "nyafuu",
 }