2015-10-31 16:50:20 +01:00
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
2017-03-17 09:42:59 +01:00
|
|
|
# Copyright 2014-2017 Mike Fährmann
|
2015-10-31 16:50:20 +01:00
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
|
# published by the Free Software Foundation.
|
|
|
|
|
2016-08-30 09:17:40 +02:00
|
|
|
"""Extract images from galleries at https://exhentai.org/"""
|
2015-10-31 16:50:20 +01:00
|
|
|
|
|
|
|
from .common import Extractor, Message
|
2017-04-25 17:12:48 +02:00
|
|
|
from .. import text, util, exception
|
2016-07-23 17:55:46 +02:00
|
|
|
from ..cache import cache
|
2014-10-12 21:56:44 +02:00
|
|
|
import time
|
|
|
|
import random
|
2016-10-11 13:27:19 +02:00
|
|
|
import requests
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2017-02-01 00:53:19 +01:00
|
|
|
|
2016-09-12 10:20:57 +02:00
|
|
|
class ExhentaiGalleryExtractor(Extractor):
|
2017-06-28 17:39:07 +02:00
|
|
|
"""Extractor for image galleries from exhentai.org"""
|
2015-11-21 04:26:30 +01:00
|
|
|
category = "exhentai"
|
2016-09-12 10:20:57 +02:00
|
|
|
subcategory = "gallery"
|
2017-09-10 22:20:47 +02:00
|
|
|
directory_fmt = ["{category}", "{gallery_id}"]
|
|
|
|
filename_fmt = "{gallery_id}_{num:>04}_{image_token}_{name}.{extension}"
|
2017-04-04 09:30:35 +02:00
|
|
|
pattern = [r"(?:https?://)?(g\.e-|e-|ex)hentai\.org/g/(\d+)/([\da-f]{10})"]
|
2016-12-31 00:51:06 +01:00
|
|
|
test = [
|
|
|
|
("https://exhentai.org/g/960460/4f0e369d82/", {
|
2017-09-24 15:59:25 +02:00
|
|
|
"keyword": "173277161e28162dcc755d2e7a88e6cd750f2477",
|
2016-12-31 00:51:06 +01:00
|
|
|
"content": "493d759de534355c9f55f8e365565b62411de146",
|
|
|
|
}),
|
|
|
|
("https://exhentai.org/g/960461/4f0e369d82/", {
|
|
|
|
"exception": exception.NotFoundError,
|
|
|
|
}),
|
|
|
|
("http://exhentai.org/g/962698/7f02358e00/", {
|
|
|
|
"exception": exception.AuthorizationError,
|
|
|
|
}),
|
|
|
|
]
|
2017-04-28 15:59:56 +02:00
|
|
|
root = "https://exhentai.org"
|
2017-07-17 10:33:36 +02:00
|
|
|
cookienames = ("ipb_member_id", "ipb_pass_hash")
|
2017-07-22 15:43:35 +02:00
|
|
|
cookiedomain = ".exhentai.org"
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2015-10-31 16:50:20 +01:00
|
|
|
def __init__(self, match):
|
|
|
|
Extractor.__init__(self)
|
2016-09-20 19:01:16 +02:00
|
|
|
self.key = {}
|
2016-10-12 15:19:31 +02:00
|
|
|
self.count = 0
|
2017-04-04 09:30:35 +02:00
|
|
|
self.version, self.gid, self.token = match.groups()
|
2017-09-24 15:59:25 +02:00
|
|
|
self.gid = util.safe_int(self.gid)
|
2017-04-25 17:12:48 +02:00
|
|
|
self.original = self.config("original", True)
|
|
|
|
self.wait_min = self.config("wait-min", 3)
|
|
|
|
self.wait_max = self.config("wait-max", 6)
|
2015-11-19 17:04:54 +01:00
|
|
|
if self.wait_max < self.wait_min:
|
|
|
|
self.wait_max = self.wait_min
|
2017-11-15 13:54:40 +01:00
|
|
|
self.session.headers["Referer"] = self.root + "/"
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2015-10-31 16:50:20 +01:00
|
|
|
def items(self):
|
2016-09-20 19:01:16 +02:00
|
|
|
self.login()
|
2015-10-31 16:50:20 +01:00
|
|
|
yield Message.Version, 1
|
2016-09-20 19:01:16 +02:00
|
|
|
|
2017-04-28 15:59:56 +02:00
|
|
|
url = "{}/g/{}/{}/".format(self.root, self.gid, self.token)
|
2017-08-05 16:11:46 +02:00
|
|
|
response = self.request(url, fatal=False)
|
2017-08-28 21:03:32 +02:00
|
|
|
page = response.text
|
2017-08-25 16:44:11 +02:00
|
|
|
|
2016-12-22 12:42:41 +01:00
|
|
|
if response.status_code == 404 and "Gallery Not Available" in page:
|
|
|
|
raise exception.AuthorizationError()
|
2017-08-28 21:03:32 +02:00
|
|
|
if self._is_sadpanda(response):
|
2017-08-25 16:44:11 +02:00
|
|
|
self.log.info("sadpanda.jpg")
|
|
|
|
raise exception.AuthorizationError()
|
2017-02-15 03:36:46 +01:00
|
|
|
if page.startswith(("Key missing", "Gallery not found")):
|
2016-08-30 09:17:40 +02:00
|
|
|
raise exception.NotFoundError("gallery")
|
2017-08-25 16:44:11 +02:00
|
|
|
|
2016-09-20 19:01:16 +02:00
|
|
|
data = self.get_job_metadata(page)
|
2017-09-24 15:59:25 +02:00
|
|
|
self.count = data["count"]
|
2015-10-31 16:50:20 +01:00
|
|
|
yield Message.Directory, data
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2016-09-20 19:01:16 +02:00
|
|
|
for url, image in self.get_images(page):
|
|
|
|
data.update(image)
|
2016-09-19 16:13:26 +02:00
|
|
|
if "/fullimg.php" in url:
|
2016-09-30 16:43:43 +02:00
|
|
|
data["extension"] = ""
|
2017-08-28 21:03:32 +02:00
|
|
|
self.wait(1.5)
|
2016-09-20 19:01:16 +02:00
|
|
|
yield Message.Url, url, data
|
|
|
|
|
2015-10-31 16:50:20 +01:00
|
|
|
def get_job_metadata(self, page):
|
2015-11-03 00:10:30 +01:00
|
|
|
"""Collect metadata for extractor-job"""
|
|
|
|
data = {
|
2017-09-10 22:20:47 +02:00
|
|
|
"gallery_id" : self.gid,
|
|
|
|
"gallery_token": self.token,
|
2015-11-03 00:10:30 +01:00
|
|
|
}
|
2016-09-20 19:01:16 +02:00
|
|
|
text.extract_all(page, (
|
|
|
|
("title" , '<h1 id="gn">', '</h1>'),
|
|
|
|
("title_jp" , '<h1 id="gj">', '</h1>'),
|
|
|
|
("date" , '>Posted:</td><td class="gdt2">', '</td>'),
|
|
|
|
("language" , '>Language:</td><td class="gdt2">', ' '),
|
|
|
|
("size" , '>File Size:</td><td class="gdt2">', ' '),
|
2017-09-10 22:20:47 +02:00
|
|
|
("size_units", '', '<'),
|
2016-09-20 19:01:16 +02:00
|
|
|
("count" , '>Length:</td><td class="gdt2">', ' '),
|
2015-11-03 00:10:30 +01:00
|
|
|
), values=data)
|
2017-03-28 13:12:44 +02:00
|
|
|
data["lang"] = util.language_to_code(data["language"])
|
2016-08-31 10:20:46 +02:00
|
|
|
data["title"] = text.unescape(data["title"])
|
|
|
|
data["title_jp"] = text.unescape(data["title_jp"])
|
2017-09-24 15:59:25 +02:00
|
|
|
data["count"] = util.safe_int(data["count"])
|
2016-09-20 19:01:16 +02:00
|
|
|
return data
|
2014-10-12 21:56:44 +02:00
|
|
|
|
2016-09-20 19:01:16 +02:00
|
|
|
def get_images(self, page):
|
2015-11-03 00:10:30 +01:00
|
|
|
"""Collect url and metadata for all images in this gallery"""
|
2017-02-01 00:53:19 +01:00
|
|
|
part = text.extract(page, 'hentai.org/s/', '"')[0]
|
2017-04-28 15:59:56 +02:00
|
|
|
yield self.image_from_page(self.root + "/s/" + part)
|
2016-09-20 19:01:16 +02:00
|
|
|
yield from self.images_from_api()
|
|
|
|
|
|
|
|
def image_from_page(self, url):
|
|
|
|
"""Get image url and data from webpage"""
|
2015-11-19 17:04:54 +01:00
|
|
|
self.wait()
|
2015-10-31 16:50:20 +01:00
|
|
|
page = self.request(url).text
|
2016-09-20 19:01:16 +02:00
|
|
|
data = text.extract_all(page, (
|
|
|
|
(None , '<div id="i3"><a onclick="return load_image(', ''),
|
|
|
|
("nextkey" , "'", "'"),
|
|
|
|
("url" , '<img id="img" src="', '"'),
|
2017-04-28 15:59:56 +02:00
|
|
|
("origurl" , 'hentai.org/fullimg.php', '"'),
|
2016-09-20 19:01:16 +02:00
|
|
|
("startkey", 'var startkey="', '";'),
|
|
|
|
("showkey" , 'var showkey="', '";'),
|
|
|
|
))[0]
|
|
|
|
self.key["start"] = data["startkey"]
|
2017-02-01 00:53:19 +01:00
|
|
|
self.key["show"] = data["showkey"]
|
|
|
|
self.key["next"] = data["nextkey"]
|
|
|
|
|
|
|
|
if self.original and data["origurl"]:
|
|
|
|
part = text.unescape(data["origurl"])
|
2017-04-28 15:59:56 +02:00
|
|
|
url = self.root + "/fullimg.php" + part
|
2017-02-01 00:53:19 +01:00
|
|
|
else:
|
|
|
|
url = data["url"]
|
|
|
|
|
2016-09-20 19:01:16 +02:00
|
|
|
return url, text.nameext_from_url(data["url"], {
|
|
|
|
"num": 1,
|
2017-09-10 22:20:47 +02:00
|
|
|
"image_token": data["startkey"],
|
2016-09-20 19:01:16 +02:00
|
|
|
})
|
2014-10-15 16:17:59 +02:00
|
|
|
|
2016-09-20 19:01:16 +02:00
|
|
|
def images_from_api(self):
|
|
|
|
"""Get image url and data from api calls"""
|
2017-04-28 15:59:56 +02:00
|
|
|
api_url = self.root + "/api.php"
|
2017-02-01 00:53:19 +01:00
|
|
|
nextkey = self.key["next"]
|
2014-10-12 21:56:44 +02:00
|
|
|
request = {
|
|
|
|
"method" : "showpage",
|
2017-09-24 15:59:25 +02:00
|
|
|
"gid" : self.gid,
|
2016-09-20 19:01:16 +02:00
|
|
|
"imgkey" : nextkey,
|
|
|
|
"showkey": self.key["show"],
|
2014-10-12 21:56:44 +02:00
|
|
|
}
|
2017-02-01 00:53:19 +01:00
|
|
|
for request["page"] in range(2, self.count + 1):
|
2016-10-11 13:27:19 +02:00
|
|
|
while True:
|
|
|
|
try:
|
2016-10-12 15:19:31 +02:00
|
|
|
self.wait()
|
2017-04-28 15:59:56 +02:00
|
|
|
page = self.session.post(api_url, json=request).json()
|
2016-10-11 13:27:19 +02:00
|
|
|
break
|
2016-10-12 15:19:31 +02:00
|
|
|
except requests.exceptions.ConnectionError:
|
|
|
|
pass
|
2016-09-20 19:01:16 +02:00
|
|
|
imgkey = nextkey
|
|
|
|
nextkey, pos = text.extract(page["i3"], "'", "'")
|
2017-02-01 00:53:19 +01:00
|
|
|
imgurl , pos = text.extract(page["i3"], 'id="img" src="', '"', pos)
|
2016-09-20 19:01:16 +02:00
|
|
|
origurl, pos = text.extract(page["i7"], '<a href="', '"')
|
2017-02-01 00:53:19 +01:00
|
|
|
|
|
|
|
if self.original and origurl:
|
|
|
|
url = text.unescape(origurl)
|
|
|
|
else:
|
|
|
|
url = imgurl
|
|
|
|
|
2016-09-20 19:01:16 +02:00
|
|
|
yield url, text.nameext_from_url(imgurl, {
|
|
|
|
"num": request["page"],
|
2017-09-10 22:20:47 +02:00
|
|
|
"image_token": imgkey
|
2016-09-20 19:01:16 +02:00
|
|
|
})
|
|
|
|
request["imgkey"] = nextkey
|
2015-11-19 17:04:54 +01:00
|
|
|
|
|
|
|
def wait(self, waittime=None):
|
|
|
|
"""Wait for a randomly chosen amount of seconds"""
|
|
|
|
if not waittime:
|
|
|
|
waittime = random.uniform(self.wait_min, self.wait_max)
|
|
|
|
else:
|
2017-08-28 21:03:32 +02:00
|
|
|
waittime = random.uniform(waittime * 0.66, waittime * 1.33)
|
2015-11-19 17:04:54 +01:00
|
|
|
time.sleep(waittime)
|
2016-07-23 17:55:46 +02:00
|
|
|
|
|
|
|
def login(self):
|
|
|
|
"""Login and set necessary cookies"""
|
2017-07-22 15:43:35 +02:00
|
|
|
if self._check_cookies(self.cookienames):
|
2017-07-17 10:33:36 +02:00
|
|
|
return
|
2017-07-25 14:59:41 +02:00
|
|
|
username, password = self._get_auth_info()
|
2017-04-28 15:59:56 +02:00
|
|
|
if not username:
|
|
|
|
self.log.info("no username given; using e-hentai.org")
|
|
|
|
self.root = "https://e-hentai.org"
|
|
|
|
self.original = False
|
|
|
|
return
|
2017-01-08 17:33:25 +01:00
|
|
|
cookies = self._login_impl(username, password)
|
2016-07-23 17:55:46 +02:00
|
|
|
for key, value in cookies.items():
|
2017-02-01 00:53:19 +01:00
|
|
|
self.session.cookies.set(
|
2017-07-22 15:43:35 +02:00
|
|
|
key, value, domain=self.cookiedomain)
|
2016-07-23 17:55:46 +02:00
|
|
|
|
2017-03-17 09:42:59 +01:00
|
|
|
@cache(maxage=90*24*60*60, keyarg=1)
|
2017-01-08 17:33:25 +01:00
|
|
|
def _login_impl(self, username, password):
|
2016-07-23 17:55:46 +02:00
|
|
|
"""Actual login implementation"""
|
2017-03-17 09:42:59 +01:00
|
|
|
self.log.info("Logging in as %s", username)
|
2016-07-23 17:55:46 +02:00
|
|
|
url = "https://forums.e-hentai.org/index.php?act=Login&CODE=01"
|
2017-08-05 16:11:46 +02:00
|
|
|
data = {
|
2016-07-23 17:55:46 +02:00
|
|
|
"CookieDate": "1",
|
|
|
|
"b": "d",
|
|
|
|
"bt": "1-1",
|
2017-01-08 17:33:25 +01:00
|
|
|
"UserName": username,
|
|
|
|
"PassWord": password,
|
2016-07-23 17:55:46 +02:00
|
|
|
"ipb_login_submit": "Login!",
|
|
|
|
}
|
2017-08-25 16:44:11 +02:00
|
|
|
headers = {
|
2017-08-29 22:10:38 +02:00
|
|
|
"Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1"
|
2017-08-25 16:44:11 +02:00
|
|
|
}
|
|
|
|
response = self.request(url, method="POST", data=data, headers=headers)
|
2016-07-23 17:55:46 +02:00
|
|
|
|
2017-08-29 22:10:38 +02:00
|
|
|
if "You are now logged in as:" not in response.text:
|
2016-07-23 17:55:46 +02:00
|
|
|
raise exception.AuthenticationError()
|
2017-08-29 22:10:38 +02:00
|
|
|
return {c: response.cookies[c] for c in self.cookienames}
|
2017-08-28 21:03:32 +02:00
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def _is_sadpanda(response):
|
|
|
|
"""Return True if the response object contains a sad panda"""
|
|
|
|
return (
|
|
|
|
response.headers.get("Content-Length") == "9615" and
|
|
|
|
"sadpanda.jpg" in response.headers.get("Content-Disposition", "")
|
|
|
|
)
|