mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-23 03:02:50 +01:00
250 lines
8.3 KiB
Python
250 lines
8.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2015-2022 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Extractors for https://hitomi.la/"""
|
|
|
|
from .common import GalleryExtractor, Extractor, Message
|
|
from .nozomi import decode_nozomi
|
|
from ..cache import memcache
|
|
from .. import text, util
|
|
import string
|
|
import json
|
|
import re
|
|
|
|
|
|
class HitomiGalleryExtractor(GalleryExtractor):
|
|
"""Extractor for image galleries from hitomi.la"""
|
|
category = "hitomi"
|
|
root = "https://hitomi.la"
|
|
pattern = (r"(?:https?://)?hitomi\.la"
|
|
r"/(?:manga|doujinshi|cg|gamecg|galleries|reader)"
|
|
r"/(?:[^/?#]+-)?(\d+)")
|
|
test = (
|
|
("https://hitomi.la/galleries/867789.html", {
|
|
"pattern": r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+"
|
|
r"/[0-9a-f]{64}\.webp",
|
|
"keyword": "4b584d09d535694d7d757c47daf5c15d116420d2",
|
|
"options": (("metadata", True),),
|
|
"count": 16,
|
|
}),
|
|
# download test
|
|
("https://hitomi.la/galleries/1401410.html", {
|
|
"range": "1",
|
|
"content": "d75d5a3d1302a48469016b20e53c26b714d17745",
|
|
}),
|
|
# Game CG with scenes (#321)
|
|
("https://hitomi.la/galleries/733697.html", {
|
|
"count": 210,
|
|
}),
|
|
# fallback for galleries only available through /reader/ URLs
|
|
("https://hitomi.la/galleries/1045954.html", {
|
|
"count": 1413,
|
|
}),
|
|
# gallery with "broken" redirect
|
|
("https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", {
|
|
"count": 10,
|
|
"options": (("format", "original"),),
|
|
"pattern": r"https://[a-c]b\.hitomi\.la/images/\d+/\d+"
|
|
r"/[0-9a-f]{64}\.jpg",
|
|
}),
|
|
# no tags
|
|
("https://hitomi.la/cg/1615823.html", {
|
|
"count": 22,
|
|
"options": (("format", "avif"),),
|
|
"pattern": r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+"
|
|
r"/[0-9a-f]{64}\.avif",
|
|
}),
|
|
("https://hitomi.la/manga/amazon-no-hiyaku-867789.html"),
|
|
("https://hitomi.la/manga/867789.html"),
|
|
("https://hitomi.la/doujinshi/867789.html"),
|
|
("https://hitomi.la/cg/867789.html"),
|
|
("https://hitomi.la/gamecg/867789.html"),
|
|
("https://hitomi.la/reader/867789.html"),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
gid = match.group(1)
|
|
url = "https://ltn.hitomi.la/galleries/{}.js".format(gid)
|
|
GalleryExtractor.__init__(self, match, url)
|
|
self.info = None
|
|
self.session.headers["Referer"] = "{}/reader/{}.html".format(
|
|
self.root, gid)
|
|
|
|
def metadata(self, page):
|
|
self.info = info = json.loads(page.partition("=")[2])
|
|
|
|
data = self._data_from_gallery_info(info)
|
|
if self.config("metadata", False):
|
|
data.update(self._data_from_gallery_page(info))
|
|
return data
|
|
|
|
def _data_from_gallery_info(self, info):
|
|
language = info.get("language")
|
|
if language:
|
|
language = language.capitalize()
|
|
|
|
date = info.get("date")
|
|
if date:
|
|
date += ":00"
|
|
|
|
tags = []
|
|
for tinfo in info.get("tags") or ():
|
|
tag = string.capwords(tinfo["tag"])
|
|
if tinfo.get("female"):
|
|
tag += " ♀"
|
|
elif tinfo.get("male"):
|
|
tag += " ♂"
|
|
tags.append(tag)
|
|
|
|
return {
|
|
"gallery_id": text.parse_int(info["id"]),
|
|
"title" : info["title"],
|
|
"type" : info["type"].capitalize(),
|
|
"language" : language,
|
|
"lang" : util.language_to_code(language),
|
|
"date" : text.parse_datetime(date, "%Y-%m-%d %H:%M:%S%z"),
|
|
"tags" : tags,
|
|
}
|
|
|
|
def _data_from_gallery_page(self, info):
|
|
url = "{}/galleries/{}.html".format(self.root, info["id"])
|
|
|
|
# follow redirects
|
|
while True:
|
|
response = self.request(url, fatal=False)
|
|
if b"<title>Redirect</title>" not in response.content:
|
|
break
|
|
url = text.extract(
|
|
response.text, 'http-equiv="refresh" content="', '"',
|
|
)[0].partition("=")[2]
|
|
|
|
if response.status_code >= 400:
|
|
return {}
|
|
|
|
def prep(value):
|
|
return [
|
|
text.unescape(string.capwords(v))
|
|
for v in text.extract_iter(value or "", '.html">', '<')
|
|
]
|
|
|
|
extr = text.extract_from(response.text)
|
|
return {
|
|
"artist" : prep(extr('<h2>', '</h2>')),
|
|
"group" : prep(extr('<td>Group</td><td>', '</td>')),
|
|
"parody" : prep(extr('<td>Series</td><td>', '</td>')),
|
|
"characters": prep(extr('<td>Characters</td><td>', '</td>')),
|
|
}
|
|
|
|
def images(self, _):
|
|
# see https://ltn.hitomi.la/gg.js
|
|
gg_m, gg_b, gg_default = _parse_gg(self)
|
|
|
|
fmt = self.config("format") or "webp"
|
|
if fmt == "original":
|
|
subdomain, fmt, ext = "b", "images", None
|
|
else:
|
|
subdomain, ext = "a", fmt
|
|
|
|
result = []
|
|
for image in self.info["files"]:
|
|
ihash = image["hash"]
|
|
idata = text.nameext_from_url(image["name"])
|
|
if ext:
|
|
idata["extension"] = ext
|
|
|
|
# see https://ltn.hitomi.la/common.js
|
|
inum = int(ihash[-1] + ihash[-3:-1], 16)
|
|
url = "https://{}{}.hitomi.la/{}/{}/{}/{}.{}".format(
|
|
chr(97 + gg_m.get(inum, gg_default)),
|
|
subdomain, fmt, gg_b, inum, ihash, idata["extension"],
|
|
)
|
|
result.append((url, idata))
|
|
return result
|
|
|
|
|
|
class HitomiTagExtractor(Extractor):
|
|
"""Extractor for galleries from tag searches on hitomi.la"""
|
|
category = "hitomi"
|
|
subcategory = "tag"
|
|
root = "https://hitomi.la"
|
|
pattern = (r"(?:https?://)?hitomi\.la/"
|
|
r"(tag|artist|group|series|type|character)/"
|
|
r"([^/?#]+)\.html")
|
|
test = (
|
|
("https://hitomi.la/tag/screenshots-japanese.html", {
|
|
"pattern": HitomiGalleryExtractor.pattern,
|
|
"count": ">= 35",
|
|
}),
|
|
("https://hitomi.la/artist/a1-all-1.html"),
|
|
("https://hitomi.la/group/initial%2Dg-all-1.html"),
|
|
("https://hitomi.la/series/amnesia-all-1.html"),
|
|
("https://hitomi.la/type/doujinshi-all-1.html"),
|
|
("https://hitomi.la/character/a2-all-1.html"),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
Extractor.__init__(self, match)
|
|
self.type, self.tag = match.groups()
|
|
|
|
tag, _, num = self.tag.rpartition("-")
|
|
if num.isdecimal():
|
|
self.tag = tag
|
|
|
|
def items(self):
|
|
data = {"_extractor": HitomiGalleryExtractor}
|
|
nozomi_url = "https://ltn.hitomi.la/{}/{}.nozomi".format(
|
|
self.type, self.tag)
|
|
headers = {
|
|
"Origin": self.root,
|
|
"Cache-Control": "max-age=0",
|
|
}
|
|
|
|
offset = 0
|
|
while True:
|
|
headers["Referer"] = "{}/{}/{}.html?page={}".format(
|
|
self.root, self.type, self.tag, offset // 100 + 1)
|
|
headers["Range"] = "bytes={}-{}".format(offset, offset+99)
|
|
nozomi = self.request(nozomi_url, headers=headers).content
|
|
|
|
for gallery_id in decode_nozomi(nozomi):
|
|
gallery_url = "{}/galleries/{}.html".format(
|
|
self.root, gallery_id)
|
|
yield Message.Queue, gallery_url, data
|
|
|
|
if len(nozomi) < 100:
|
|
return
|
|
offset += 100
|
|
|
|
|
|
@memcache()
|
|
def _parse_gg(extr):
|
|
page = extr.request("https://ltn.hitomi.la/gg.js").text
|
|
|
|
m = {}
|
|
|
|
keys = []
|
|
for match in re.finditer(
|
|
r"case\s+(\d+):(?:\s*o\s*=\s*(\d+))?", page):
|
|
key, value = match.groups()
|
|
keys.append(int(key))
|
|
|
|
if value:
|
|
value = int(value)
|
|
for key in keys:
|
|
m[key] = value
|
|
keys.clear()
|
|
|
|
for match in re.finditer(
|
|
r"if\s+\(g\s*===?\s*(\d+)\)[\s{]*o\s*=\s*(\d+)", page):
|
|
m[int(match.group(1))] = int(match.group(2))
|
|
|
|
d = re.search(r"(?:var\s|default:)\s*o\s*=\s*(\d+)", page)
|
|
b = re.search(r"b:\s*[\"'](.+)[\"']", page)
|
|
|
|
return m, b.group(1).strip("/"), int(d.group(1)) if d else 1
|