Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-25 04:02:32 +01:00
Code Issues Releases Wiki Activity

[reactor] inherit from BaseExtractor

Browse Source
This commit is contained in:
Mike Fährmann 2021-11-25 16:18:48 +01:00
parent f4ea216c95
commit d8d9502e1e
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
3 changed files with 116 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
docs/supportedsites.md
View File

@ -379,12 +379,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Publications, User Profiles</td>
<td></td>
</tr>
<tr>
<td>Joyreactor</td>
<td>http://joyreactor.cc/</td>
<td>Posts, Search Results, Tag Searches, User Profiles</td>
<td></td>
</tr>
<tr>
<td>Keenspot</td>
<td>http://www.keenspot.com/</td>
@ -619,12 +613,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Galleries, User Profiles</td>
<td></td>
</tr>
<tr>
<td>Pornreactor</td>
<td>http://pornreactor.cc/</td>
<td>Posts, Search Results, Tag Searches, User Profiles</td>
<td></td>
</tr>
<tr>
<td>Postimg</td>
<td>https://postimages.org/</td>
@ -950,6 +938,28 @@ Consider all sites to be NSFW unless otherwise known.
<td>API Key</td>
</tr>
<tr>
<td colspan="4"><strong>Reactor Instances</strong></td>
</tr>
<tr>
<td>JoyReactor</td>
<td>http://joyreactor.cc/</td>
<td>Posts, Search Results, Tag Searches, User Profiles</td>
<td></td>
</tr>
<tr>
<td>PornReactor</td>
<td>http://pornreactor.cc/</td>
<td>Posts, Search Results, Tag Searches, User Profiles</td>
<td></td>
</tr>
<tr>
<td>Reactor</td>
<td>http://reactor.cc/</td>
<td>Posts, Search Results, Tag Searches, User Profiles</td>
<td></td>
</tr>
<tr>
<td colspan="4"><strong>Moebooru and MyImouto</strong></td>
</tr>

225
gallery_dl/extractor/reactor.py
View File

@ -8,29 +8,29 @@
"""Generic extractors for *reactor sites"""
from .common import Extractor, Message
from .common import BaseExtractor, Message
from .. import text
import urllib.parse
import json
BASE_PATTERN = r"(?:https?://)?((?:[^/.]+\.)?reactor\.cc)"
class ReactorExtractor(Extractor):
class ReactorExtractor(BaseExtractor):
"""Base class for *reactor.cc extractors"""
basecategory = "reactor"
filename_fmt = "{post_id}_{num:>02}{title[:100]:?_//}.{extension}"
archive_fmt = "{post_id}_{num}"
instances = ()
request_interval = 5.0
def __init__(self, match):
Extractor.__init__(self, match)
self.root = "http://" + match.group(1)
BaseExtractor.__init__(self, match)
url = text.ensure_http_scheme(match.group(0), "http://")
pos = url.index("/", 10)
self.root, self.path = url[:pos], url[pos:]
self.session.headers["Referer"] = self.root
self.gif = self.config("gif", False)
if not self.category:
if self.category == "reactor":
# set category based on domain name
netloc = urllib.parse.urlsplit(self.root).netloc
self.category = netloc.rpartition(".")[0]
@ -50,7 +50,7 @@ class ReactorExtractor(Extractor):
def posts(self):
"""Return all relevant post-objects"""
return self._pagination(self.url)
return self._pagination(self.root + self.path)
def _pagination(self, url):
while True:
@ -145,29 +145,78 @@ class ReactorExtractor(Extractor):
}
BASE_PATTERN = ReactorExtractor.update({
"reactor" : {
"root": "http://reactor.cc",
"pattern": r"(?:[^/.]+\.)?reactor\.cc",
},
"joyreactor" : {
"root": "http://joyreactor.cc",
"pattern": r"(?:www\.)?joyreactor\.c(?:c|om)",
},
"pornreactor": {
"root": "http://pornreactor.cc",
"pattern": r"(?:www\.)?(?:pornreactor\.cc|fapreactor.com)",
},
})
class ReactorTagExtractor(ReactorExtractor):
"""Extractor for tag searches on *reactor.cc sites"""
subcategory = "tag"
directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "{search_tags}_{post_id}_{num}"
pattern = BASE_PATTERN + r"/tag/([^/?#]+)"
test = ("http://anime.reactor.cc/tag/Anime+Art",)
test = (
("http://reactor.cc/tag/gif"),
("http://anime.reactor.cc/tag/Anime+Art"),
("http://joyreactor.cc/tag/Advent+Cirno", {
"count": ">= 15",
}),
("http://joyreactor.com/tag/Cirno", {
"url": "aa59090590b26f4654881301fe8fe748a51625a8",
}),
("http://pornreactor.cc/tag/RiceGnat", {
"range": "1-25",
"count": ">= 25",
}),
("http://fapreactor.com/tag/RiceGnat"),
)
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.tag = match.group(2)
self.tag = match.group(match.lastindex)
def metadata(self):
return {"search_tags": text.unescape(self.tag).replace("+", " ")}
class ReactorSearchExtractor(ReactorTagExtractor):
class ReactorSearchExtractor(ReactorExtractor):
"""Extractor for search results on *reactor.cc sites"""
subcategory = "search"
directory_fmt = ("{category}", "search", "{search_tags}")
archive_fmt = "s_{search_tags}_{post_id}_{num}"
pattern = BASE_PATTERN + r"/search(?:/|\?q=)([^/?#]+)"
test = ("http://anime.reactor.cc/search?q=Art",)
test = (
("http://reactor.cc/search?q=Art"),
("http://joyreactor.cc/search/Nature", {
"range": "1-25",
"count": ">= 20",
}),
("http://joyreactor.com/search?q=Nature", {
"range": "1-25",
"count": ">= 20",
}),
("http://pornreactor.cc/search?q=ecchi+hentai"),
("http://fapreactor.com/search/ecchi+hentai"),
)
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.tag = match.group(match.lastindex)
def metadata(self):
return {"search_tags": text.unescape(self.tag).replace("+", " ")}
class ReactorUserExtractor(ReactorExtractor):
@ -175,11 +224,23 @@ class ReactorUserExtractor(ReactorExtractor):
subcategory = "user"
directory_fmt = ("{category}", "user", "{user}")
pattern = BASE_PATTERN + r"/user/([^/?#]+)"
test = ("http://anime.reactor.cc/user/Shuster",)
test = (
("http://reactor.cc/user/Dioklet"),
("http://anime.reactor.cc/user/Shuster"),
("http://joyreactor.cc/user/hemantic"),
("http://joyreactor.com/user/Tacoman123", {
"url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5",
}),
("http://pornreactor.cc/user/Disillusion", {
"range": "1-25",
"count": ">= 20",
}),
("http://fapreactor.com/user/Disillusion"),
)
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.user = match.group(2)
self.user = match.group(match.lastindex)
def metadata(self):
return {"user": text.unescape(self.user).replace("+", " ")}
@ -189,75 +250,11 @@ class ReactorPostExtractor(ReactorExtractor):
"""Extractor for single posts on *reactor.cc sites"""
subcategory = "post"
pattern = BASE_PATTERN + r"/post/(\d+)"
test = ("http://anime.reactor.cc/post/3576250",)
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.post_id = match.group(2)
def items(self):
post = self.request(self.url).text
pos = post.find('class="uhead">')
for image in self._parse_post(post[pos:]):
if image["num"] == 1:
yield Message.Directory, image
url = image["url"]
yield Message.Url, url, text.nameext_from_url(url, image)
# --------------------------------------------------------------------
# JoyReactor
JR_BASE_PATTERN = r"(?:https?://)?(?:www\.)?(joyreactor\.c(?:c|om))"
class JoyreactorTagExtractor(ReactorTagExtractor):
"""Extractor for tag searches on joyreactor.cc"""
category = "joyreactor"
pattern = JR_BASE_PATTERN + r"/tag/([^/?#]+)"
test = (
("http://joyreactor.cc/tag/Advent+Cirno", {
"count": ">= 15",
("http://reactor.cc/post/4999736", {
"url": "dfc74d150d7267384d8c229c4b82aa210755daa0",
}),
("http://joyreactor.com/tag/Cirno", {
"url": "aa59090590b26f4654881301fe8fe748a51625a8",
}),
)
class JoyreactorSearchExtractor(ReactorSearchExtractor):
"""Extractor for search results on joyreactor.cc"""
category = "joyreactor"
pattern = JR_BASE_PATTERN + r"/search(?:/|\?q=)([^/?#]+)"
test = (
("http://joyreactor.cc/search/Nature", {
"range": "1-25",
"count": ">= 20",
}),
("http://joyreactor.com/search?q=Nature", {
"range": "1-25",
"count": ">= 20",
}),
)
class JoyreactorUserExtractor(ReactorUserExtractor):
"""Extractor for all posts of a user on joyreactor.cc"""
category = "joyreactor"
pattern = JR_BASE_PATTERN + r"/user/([^/?#]+)"
test = (
("http://joyreactor.cc/user/hemantic"),
("http://joyreactor.com/user/Tacoman123", {
"url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5",
}),
)
class JoyreactorPostExtractor(ReactorPostExtractor):
"""Extractor for single posts on joyreactor.cc"""
category = "joyreactor"
pattern = JR_BASE_PATTERN + r"/post/(\d+)"
test = (
("http://anime.reactor.cc/post/3576250"),
("http://joyreactor.com/post/3721876", { # single image
"pattern": r"http://img\d\.joyreactor\.com/pics/post/full"
r"/cartoon-painting-monster-lake-4841316.jpeg",
@ -281,57 +278,6 @@ class JoyreactorPostExtractor(ReactorPostExtractor):
("http://joyreactor.cc/post/1299", { # "malformed" JSON
"url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39",
}),
)
# --------------------------------------------------------------------
# PornReactor
PR_BASE_PATTERN = r"(?:https?://)?(?:www\.)?(pornreactor\.cc|fapreactor.com)"
class PornreactorTagExtractor(ReactorTagExtractor):
"""Extractor for tag searches on pornreactor.cc"""
category = "pornreactor"
pattern = PR_BASE_PATTERN + r"/tag/([^/?#]+)"
test = (
("http://pornreactor.cc/tag/RiceGnat", {
"range": "1-25",
"count": ">= 25",
}),
("http://fapreactor.com/tag/RiceGnat"),
)
class PornreactorSearchExtractor(ReactorSearchExtractor):
"""Extractor for search results on pornreactor.cc"""
category = "pornreactor"
pattern = PR_BASE_PATTERN + r"/search(?:/|\?q=)([^/?#]+)"
test = (
("http://pornreactor.cc/search?q=ecchi+hentai"),
("http://fapreactor.com/search/ecchi+hentai"),
)
class PornreactorUserExtractor(ReactorUserExtractor):
"""Extractor for all posts of a user on pornreactor.cc"""
category = "pornreactor"
pattern = PR_BASE_PATTERN + r"/user/([^/?#]+)"
test = (
("http://pornreactor.cc/user/Disillusion", {
"range": "1-25",
"count": ">= 20",
}),
("http://fapreactor.com/user/Disillusion"),
)
class PornreactorPostExtractor(ReactorPostExtractor):
"""Extractor for single posts on pornreactor.cc"""
category = "pornreactor"
subcategory = "post"
pattern = PR_BASE_PATTERN + r"/post/(\d+)"
test = (
("http://pornreactor.cc/post/863166", {
"url": "a09fb0577489e1f9564c25d0ad576f81b19c2ef3",
"content": "ec6b0568bfb1803648744077da082d14de844340",
@ -340,3 +286,16 @@ class PornreactorPostExtractor(ReactorPostExtractor):
"url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54",
}),
)
def __init__(self, match):
ReactorExtractor.__init__(self, match)
self.post_id = match.group(match.lastindex)
def items(self):
post = self.request(self.root + self.path).text
pos = post.find('class="uhead">')
for image in self._parse_post(post[pos:]):
if image["num"] == 1:
yield Message.Directory, image
url = image["url"]
yield Message.Url, url, text.nameext_from_url(url, image)

2
scripts/supportedsites.py
View File

@ -54,6 +54,7 @@ CATEGORY_MAP = {
"imagechest" : "ImageChest",
"imgth" : "imgth",
"imgur" : "imgur",
"joyreactor" : "JoyReactor",
"kabeuchi" : "かべうち",
"kireicake" : "Kirei Cake",
"lineblog" : "LINE BLOG",
@ -78,6 +79,7 @@ CATEGORY_MAP = {
"paheal" : "rule #34",
"photovogue" : "PhotoVogue",
"pornimagesxxx" : "Porn Image",
"pornreactor" : "PornReactor",
"powermanga" : "PowerManga",
"readcomiconline": "Read Comic Online",
"rbt" : "RebeccaBlackTech",