mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-25 04:02:32 +01:00
[wikiart] add extractors (#179)
for - artists: https://www.wikiart.org/en/thomas-cole - artist-listings: https://www.wikiart.org/en/artists-by-century/12 - artwork-listings: https://www.wikiart.org/en/paintings-by-media/grisaille
This commit is contained in:
parent
9ebd29fcc1
commit
c70b21248d
@ -93,6 +93,7 @@ Twitter https://twitter.com/ Media Timelines, Timeli
|
|||||||
Wallhaven https://alpha.wallhaven.cc/ individual Images, Search Results Optional
|
Wallhaven https://alpha.wallhaven.cc/ individual Images, Search Results Optional
|
||||||
Warosu https://warosu.org/ Threads
|
Warosu https://warosu.org/ Threads
|
||||||
Weibo https://www.weibo.com/ Images from Users, Images from Statuses
|
Weibo https://www.weibo.com/ Images from Users, Images from Statuses
|
||||||
|
WikiArt.org https://www.wikiart.org/ Artists, Artworks
|
||||||
World Three http://www.slide.world-three.org/ Chapters, Manga
|
World Three http://www.slide.world-three.org/ Chapters, Manga
|
||||||
XVideos https://www.xvideos.com/ Images from Users, Galleries
|
XVideos https://www.xvideos.com/ Images from Users, Galleries
|
||||||
Yandere https://yande.re/ Pools, Popular Images, Posts, Tag-Searches
|
Yandere https://yande.re/ Pools, Popular Images, Posts, Tag-Searches
|
||||||
|
@ -83,6 +83,7 @@ modules = [
|
|||||||
"wallhaven",
|
"wallhaven",
|
||||||
"warosu",
|
"warosu",
|
||||||
"weibo",
|
"weibo",
|
||||||
|
"wikiart",
|
||||||
"yandere",
|
"yandere",
|
||||||
"xvideos",
|
"xvideos",
|
||||||
"yuki",
|
"yuki",
|
||||||
|
135
gallery_dl/extractor/wikiart.py
Normal file
135
gallery_dl/extractor/wikiart.py
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright 2019 Mike Fährmann
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License version 2 as
|
||||||
|
# published by the Free Software Foundation.
|
||||||
|
|
||||||
|
"""Extractors for https://www.wikiart.org/"""
|
||||||
|
|
||||||
|
from .common import Extractor, Message
|
||||||
|
from .. import text
|
||||||
|
|
||||||
|
BASE_PATTERN = r"(?:https://)?(?:www\.)?wikiart\.org/([a-z]+)"
|
||||||
|
|
||||||
|
|
||||||
|
class WikiartExtractor(Extractor):
|
||||||
|
"""Base class for wikiart extractors"""
|
||||||
|
category = "wikiart"
|
||||||
|
filename_fmt = "{id}_{title}.{extension}"
|
||||||
|
archive_fmt = "{id}"
|
||||||
|
root = "https://www.wikiart.org"
|
||||||
|
|
||||||
|
def __init__(self, match):
|
||||||
|
Extractor.__init__(self, match)
|
||||||
|
self.lang = match.group(1)
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
data = self.metadata()
|
||||||
|
yield Message.Version, 1
|
||||||
|
yield Message.Directory, data
|
||||||
|
for painting in self.paintings():
|
||||||
|
url = painting["image"]
|
||||||
|
painting.update(data)
|
||||||
|
yield Message.Url, url, text.nameext_from_url(url, painting)
|
||||||
|
|
||||||
|
def metadata(self):
|
||||||
|
""" """
|
||||||
|
|
||||||
|
def paintings(self):
|
||||||
|
""" """
|
||||||
|
|
||||||
|
def _pagination(self, url, key="Paintings", extra_params=None):
|
||||||
|
headers = {
|
||||||
|
"X-Requested-With": "XMLHttpRequest",
|
||||||
|
"Referer": url,
|
||||||
|
}
|
||||||
|
params = {
|
||||||
|
"json": "2",
|
||||||
|
"layout": "new",
|
||||||
|
"page": 1,
|
||||||
|
"resultType": "masonry",
|
||||||
|
}
|
||||||
|
if extra_params:
|
||||||
|
params.update(extra_params)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
data = self.request(url, headers=headers, params=params).json()
|
||||||
|
items = data.get(key)
|
||||||
|
if not items:
|
||||||
|
return
|
||||||
|
yield from items
|
||||||
|
params["page"] += 1
|
||||||
|
|
||||||
|
|
||||||
|
class WikiartArtistExtractor(WikiartExtractor):
|
||||||
|
"""Extractor for an artist's paintings on wikiart.org"""
|
||||||
|
subcategory = "artist"
|
||||||
|
directory_fmt = ("{category}", "{artist[artistName]}")
|
||||||
|
pattern = BASE_PATTERN + r"/(?!\w+-by-)([\w-]+)"
|
||||||
|
test = ("https://www.wikiart.org/en/thomas-cole", {
|
||||||
|
"url": "f1eee8158f5b8b7380382ab730a8f53884715c8b",
|
||||||
|
"keyword": "b62678394ce645815963883d5c9642255307225f",
|
||||||
|
})
|
||||||
|
|
||||||
|
def __init__(self, match):
|
||||||
|
WikiartExtractor.__init__(self, match)
|
||||||
|
self.artist = match.group(2)
|
||||||
|
|
||||||
|
def metadata(self):
|
||||||
|
url = "{}/{}/{}?json=2".format(self.root, self.lang, self.artist)
|
||||||
|
return {"artist": self.request(url).json()}
|
||||||
|
|
||||||
|
def paintings(self):
|
||||||
|
""" """
|
||||||
|
url = "{}/{}/{}/mode/all-paintings".format(
|
||||||
|
self.root, self.lang, self.artist)
|
||||||
|
return self._pagination(url)
|
||||||
|
|
||||||
|
|
||||||
|
class WikiartArtworksExtractor(WikiartExtractor):
|
||||||
|
"""Extractor for artwork collections on wikiart.org"""
|
||||||
|
subcategory = "artworks"
|
||||||
|
directory_fmt = ("{category}", "Artworks by {group!c}", "{type}")
|
||||||
|
pattern = BASE_PATTERN + r"/paintings-by-([\w-]+)/([\w-]+)"
|
||||||
|
test = ("https://www.wikiart.org/en/paintings-by-media/grisaille", {
|
||||||
|
"url": "f92d55669fa949491c26a5437527adb14b35b8cc",
|
||||||
|
})
|
||||||
|
|
||||||
|
def __init__(self, match):
|
||||||
|
WikiartExtractor.__init__(self, match)
|
||||||
|
self.group = match.group(2)
|
||||||
|
self.type = match.group(3)
|
||||||
|
|
||||||
|
def metadata(self):
|
||||||
|
return {"group": self.group, "type": self.type}
|
||||||
|
|
||||||
|
def paintings(self):
|
||||||
|
url = "{}/{}/paintings-by-{}/{}".format(
|
||||||
|
self.root, self.lang, self.group, self.type)
|
||||||
|
return self._pagination(url)
|
||||||
|
|
||||||
|
|
||||||
|
class WikiartArtistsExtractor(WikiartExtractor):
|
||||||
|
"""Extractor for artist collections on wikiart.org"""
|
||||||
|
subcategory = "artists"
|
||||||
|
pattern = (BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)")
|
||||||
|
test = ("https://www.wikiart.org/en/artists-by-century/12", {
|
||||||
|
"pattern": WikiartArtistExtractor.pattern,
|
||||||
|
"count": 7,
|
||||||
|
})
|
||||||
|
|
||||||
|
def __init__(self, match):
|
||||||
|
WikiartExtractor.__init__(self, match)
|
||||||
|
self.group = match.group(2)
|
||||||
|
self.type = match.group(3)
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
url = "{}/{}/App/Search/Artists-by-{}".format(
|
||||||
|
self.root, self.lang, self.group)
|
||||||
|
params = {"json": "3", "searchterm": self.type}
|
||||||
|
|
||||||
|
for artist in self._pagination(url, "Artists", params):
|
||||||
|
artist["_extractor"] = WikiartArtistExtractor
|
||||||
|
yield Message.Queue, self.root + artist["artistUrl"], artist
|
@ -63,6 +63,7 @@ CATEGORY_MAP = {
|
|||||||
"slideshare" : "SlideShare",
|
"slideshare" : "SlideShare",
|
||||||
"smugmug" : "SmugMug",
|
"smugmug" : "SmugMug",
|
||||||
"thebarchive" : "The /b/ Archive",
|
"thebarchive" : "The /b/ Archive",
|
||||||
|
"wikiart" : "WikiArt.org",
|
||||||
"worldthree" : "World Three",
|
"worldthree" : "World Three",
|
||||||
"xvideos" : "XVideos",
|
"xvideos" : "XVideos",
|
||||||
"yuki" : "yuki.la 4chan archive",
|
"yuki" : "yuki.la 4chan archive",
|
||||||
@ -70,6 +71,7 @@ CATEGORY_MAP = {
|
|||||||
|
|
||||||
SUBCATEGORY_MAP = {
|
SUBCATEGORY_MAP = {
|
||||||
"artwork": "Artwork Listings",
|
"artwork": "Artwork Listings",
|
||||||
|
"artists": "",
|
||||||
"doujin" : "Doujin",
|
"doujin" : "Doujin",
|
||||||
"gallery": "Galleries",
|
"gallery": "Galleries",
|
||||||
"image" : "individual Images",
|
"image" : "individual Images",
|
||||||
|
@ -25,8 +25,9 @@ TRAVIS_SKIP = {
|
|||||||
|
|
||||||
# temporary issues, etc.
|
# temporary issues, etc.
|
||||||
BROKEN = {
|
BROKEN = {
|
||||||
|
"komikcast",
|
||||||
|
"mangahere",
|
||||||
"mangapark",
|
"mangapark",
|
||||||
"nyafuu",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user