', '<'),
"uploader" : extr('
', '
'),
"date" : text.parse_datetime(extr(
'>Posted:
', ' | '), "%Y-%m-%d %H:%M"),
"parent" : extr(
'>Parent:
Visible: | ', '<'),
"language" : extr('>Language: | ', ' '),
"filesize" : text.parse_bytes(extr(
'>File Size: | ', '<').rstrip("Bb")),
"filecount" : extr('>Length: | ', ' '),
"favorites" : extr('id="favcount">', ' '),
"rating" : extr(">Average: ", "<"),
"torrentcount" : extr('>Torrent Download (', ')'),
}
if data["uploader"].startswith("<"):
data["uploader"] = text.unescape(text.extr(
data["uploader"], ">", "<"))
f = data["favorites"][0]
if f == "N":
data["favorites"] = "0"
elif f == "O":
data["favorites"] = "1"
data["lang"] = util.language_to_code(data["language"])
data["tags"] = [
text.unquote(tag.replace("+", " "))
for tag in text.extract_iter(page, 'hentai.org/tag/', '"')
]
return data
def metadata_from_api(self):
url = self.root + "/api.php"
data = {
"method": "gdata",
"gidlist": ((self.gallery_id, self.gallery_token),),
"namespace": 1,
}
data = self.request(url, method="POST", json=data).json()
if "error" in data:
raise exception.StopExtraction(data["error"])
return data["gmetadata"][0]
def image_from_page(self, page):
"""Get image url and data from webpage"""
pos = page.index('", "")
self.log.debug("Image Limits: %s/%s", current, self.limits)
self._remaining = self.limits - text.parse_int(current)
def _gallery_page(self):
url = "{}/g/{}/{}/".format(
self.root, self.gallery_id, self.gallery_token)
response = self.request(url, fatal=False)
page = response.text
if response.status_code == 404 and "Gallery Not Available" in page:
raise exception.AuthorizationError()
if page.startswith(("Key missing", "Gallery not found")):
raise exception.NotFoundError("gallery")
if "hentai.org/mpv/" in page:
self.log.warning("Enabled Multi-Page Viewer is not supported")
return page
def _image_page(self):
url = "{}/s/{}/{}-{}".format(
self.root, self.image_token, self.gallery_id, self.image_num)
page = self.request(url, fatal=False).text
if page.startswith(("Invalid page", "Keep trying")):
raise exception.NotFoundError("image page")
return page
@staticmethod
def _parse_image_info(url):
for part in url.split("/")[4:]:
try:
_, size, width, height, _ = part.split("-")
break
except ValueError:
pass
else:
size = width = height = 0
return {
"cost" : 1,
"size" : text.parse_int(size),
"width" : text.parse_int(width),
"height": text.parse_int(height),
}
@staticmethod
def _parse_original_info(info):
parts = info.lstrip().split(" ")
size = text.parse_bytes(parts[3] + parts[4][0])
return {
# 1 initial point + 1 per 0.1 MB
"cost" : 1 + math.ceil(size / 100000),
"size" : size,
"width" : text.parse_int(parts[0]),
"height": text.parse_int(parts[2]),
}
class ExhentaiSearchExtractor(ExhentaiExtractor):
"""Extractor for exhentai search results"""
subcategory = "search"
pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))"
test = (
("https://e-hentai.org/?f_search=touhou"),
("https://exhentai.org/?f_cats=767&f_search=touhou"),
("https://exhentai.org/tag/parody:touhou+project"),
(("https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0"
"&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0"
"&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter"), {
"pattern": ExhentaiGalleryExtractor.pattern,
"range": "1-30",
"count": 30,
"keyword": {
"gallery_id": int,
"gallery_token": r"re:^[0-9a-f]{10}$"
},
}),
)
def __init__(self, match):
ExhentaiExtractor.__init__(self, match)
self.search_url = self.root
_, query, tag = match.groups()
if tag:
if "+" in tag:
ns, _, tag = tag.rpartition(":")
tag = '{}:"{}$"'.format(ns, tag.replace("+", " "))
else:
tag += "$"
self.params = {"f_search": tag, "page": 0}
else:
self.params = text.parse_query(query)
if "next" not in self.params:
self.params["page"] = text.parse_int(self.params.get("page"))
def items(self):
self.login()
data = {"_extractor": ExhentaiGalleryExtractor}
search_url = self.search_url
params = self.params
while True:
last = None
page = self.request(search_url, params=params).text
for gallery in ExhentaiGalleryExtractor.pattern.finditer(page):
url = gallery.group(0)
if url == last:
continue
last = url
data["gallery_id"] = text.parse_int(gallery.group(2))
data["gallery_token"] = gallery.group(3)
yield Message.Queue, url + "/", data
next_url = text.extr(page, 'nexturl="', '"', None)
if next_url is not None:
if not next_url:
return
search_url = next_url
params = None
elif 'class="ptdd">><' in page or ">No hits found" in page:
return
else:
params["page"] += 1
class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor):
"""Extractor for favorited exhentai galleries"""
subcategory = "favorite"
pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?"
test = (
("https://e-hentai.org/favorites.php", {
"count": 1,
"pattern": r"https?://e-hentai\.org/g/1200119/d55c44d3d0"
}),
("https://exhentai.org/favorites.php?favcat=1&f_search=touhou"
"&f_apply=Search+Favorites"),
)
def __init__(self, match):
ExhentaiSearchExtractor.__init__(self, match)
self.search_url = self.root + "/favorites.php"
|