mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-23 19:22:32 +01:00
355 lines
12 KiB
Python
355 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2019 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Extractors for https://www.tsumino.com/"""
|
|
|
|
from .common import GalleryExtractor, Extractor, Message
|
|
from .. import text, exception
|
|
from ..cache import cache
|
|
|
|
|
|
class TsuminoBase():
|
|
"""Base class for tsumino extractors"""
|
|
category = "tsumino"
|
|
cookiedomain = "www.tsumino.com"
|
|
root = "https://www.tsumino.com"
|
|
|
|
def login(self):
|
|
username, password = self._get_auth_info()
|
|
if username:
|
|
self._update_cookies(self._login_impl(username, password))
|
|
else:
|
|
self.session.cookies.setdefault(
|
|
"ASP.NET_SessionId", "x1drgggilez4cpkttneukrc5")
|
|
|
|
@cache(maxage=14*24*3600, keyarg=1)
|
|
def _login_impl(self, username, password):
|
|
self.log.info("Logging in as %s", username)
|
|
url = "{}/Account/Login".format(self.root)
|
|
headers = {"Referer": url}
|
|
data = {"Username": username, "Password": password}
|
|
|
|
response = self.request(url, method="POST", headers=headers, data=data)
|
|
if not response.history:
|
|
raise exception.AuthenticationError()
|
|
return {".aotsumino": response.history[0].cookies[".aotsumino"]}
|
|
|
|
|
|
class TsuminoGalleryExtractor(TsuminoBase, GalleryExtractor):
|
|
"""Extractor for image galleries on tsumino.com"""
|
|
pattern = (r"(?i)(?:https?://)?(?:www\.)?tsumino\.com"
|
|
r"/(?:Book/Info|Read/View)/(\d+)")
|
|
test = (
|
|
("https://www.tsumino.com/Book/Info/40996", {
|
|
"url": "84bf30a86623039fc87855680fada884dc8a1ddd",
|
|
"keyword": {
|
|
"title" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou",
|
|
"title_en" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou",
|
|
"title_jp" : "シコシコ大好きナイチンゲール + 会場限定おまけ本",
|
|
"gallery_id": 40996,
|
|
"date" : "2018 June 29",
|
|
"count" : 42,
|
|
"collection": "",
|
|
"artist" : ["Itou Life"],
|
|
"group" : ["Itou Life"],
|
|
"parody" : ["Fate/Grand Order"],
|
|
"characters": list,
|
|
"tags" : list,
|
|
"type" : "Doujinshi",
|
|
"rating" : float,
|
|
"uploader" : "sehki",
|
|
"lang" : "en",
|
|
"language" : "English",
|
|
"thumbnail" : "http://www.tsumino.com/Image/Thumb/40996",
|
|
},
|
|
}),
|
|
("https://www.tsumino.com/Read/View/45834"),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
self.gallery_id = match.group(1)
|
|
url = "{}/Book/Info/{}".format(self.root, self.gallery_id)
|
|
GalleryExtractor.__init__(self, match, url)
|
|
|
|
def metadata(self, page):
|
|
extr = text.extract
|
|
title, pos = extr(page, '"og:title" content="', '"')
|
|
thumb, pos = extr(page, '"og:image" content="', '"', pos)
|
|
title_en, _, title_jp = text.unescape(title).partition("/")
|
|
title_en = title_en.strip()
|
|
title_jp = title_jp.strip()
|
|
|
|
uploader , pos = extr(page, 'id="Uploader">' , '</div>', pos)
|
|
date , pos = extr(page, 'id="Uploaded">' , '</div>', pos)
|
|
rating , pos = extr(page, 'id="Rating">' , '</div>', pos)
|
|
gtype , pos = extr(page, 'id="Category">' , '</div>', pos)
|
|
collection, pos = extr(page, 'id="Collection">', '</div>', pos)
|
|
group , pos = extr(page, 'id="Group">' , '</div>', pos)
|
|
artist , pos = extr(page, 'id="Artist">' , '</div>', pos)
|
|
parody , pos = extr(page, 'id="Parody">' , '</div>', pos)
|
|
character , pos = extr(page, 'id="Character">' , '</div>', pos)
|
|
tags , pos = extr(page, 'id="Tag">' , '</div>', pos)
|
|
|
|
return {
|
|
"gallery_id": text.parse_int(self.gallery_id),
|
|
"title": title_en or title_jp,
|
|
"title_en": title_en,
|
|
"title_jp": title_jp,
|
|
"thumbnail": thumb,
|
|
"uploader": text.remove_html(uploader),
|
|
"date": date.strip(),
|
|
"rating": text.parse_float(rating.partition(" ")[0]),
|
|
"type": text.remove_html(gtype),
|
|
"collection": text.remove_html(collection),
|
|
"group": text.split_html(group),
|
|
"artist": text.split_html(artist),
|
|
"parody": text.split_html(parody),
|
|
"characters": text.split_html(character),
|
|
"tags": text.split_html(tags),
|
|
"language": "English",
|
|
"lang": "en",
|
|
}
|
|
|
|
def images(self, page):
|
|
url = "{}/Read/Load/?q={}".format(self.root, self.gallery_id)
|
|
headers = {"Referer": self.chapter_url}
|
|
response = self.request(url, headers=headers, expect=(404,))
|
|
|
|
if response.status_code == 404:
|
|
url = "{}/Read/View/{}".format(self.root, self.gallery_id)
|
|
self.log.error(
|
|
"Failed to get gallery JSON data. Visit '%s' in a browser "
|
|
"and solve the CAPTCHA to continue.", url)
|
|
raise exception.StopExtraction()
|
|
|
|
base = self.root + "/Image/Object?name="
|
|
return [
|
|
(base + text.quote(name), None)
|
|
for name in response.json()["reader_page_urls"]
|
|
]
|
|
|
|
|
|
class TsuminoSearchExtractor(TsuminoBase, Extractor):
|
|
"""Extractor for search results on tsumino.com"""
|
|
subcategory = "search"
|
|
pattern = (r"(?i)(?:https?://)?(?:www\.)?tsumino\.com"
|
|
r"/(?:Books/?)?#(.+)")
|
|
test = (
|
|
("https://www.tsumino.com/Books#?Character=Reimu+Hakurei", {
|
|
"pattern": TsuminoGalleryExtractor.pattern,
|
|
"range": "1-40",
|
|
"count": 40,
|
|
}),
|
|
(("http://www.tsumino.com/Books#~(Tags~(~"
|
|
"(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~"
|
|
"(Type~'1~Text~'Pantyhose~Exclude~false)))#"), {
|
|
"pattern": TsuminoGalleryExtractor.pattern,
|
|
"count": ">= 3",
|
|
}),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
Extractor.__init__(self, match)
|
|
self.query = match.group(1)
|
|
|
|
def items(self):
|
|
yield Message.Version, 1
|
|
for gallery in self.galleries():
|
|
url = "{}/Book/Info/{}".format(self.root, gallery["Id"])
|
|
gallery["_extractor"] = TsuminoGalleryExtractor
|
|
yield Message.Queue, url, gallery
|
|
|
|
def galleries(self):
|
|
"""Return all gallery results matching 'self.query'"""
|
|
url = "{}/Books/Operate".format(self.root)
|
|
headers = {
|
|
"Referer": "{}/".format(self.root),
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
}
|
|
data = {
|
|
"PageNumber": 1,
|
|
"Text": "",
|
|
"Sort": "Newest",
|
|
"List": "0",
|
|
"Length": "0",
|
|
"MinimumRating": "0",
|
|
"ExcludeList": "0",
|
|
"CompletelyExcludeHated": "false",
|
|
}
|
|
data.update(self._parse(self.query))
|
|
|
|
while True:
|
|
info = self.request(
|
|
url, method="POST", headers=headers, data=data).json()
|
|
|
|
for gallery in info["Data"]:
|
|
yield gallery["Entry"]
|
|
|
|
if info["PageNumber"] >= info["PageCount"]:
|
|
return
|
|
data["PageNumber"] += 1
|
|
|
|
def _parse(self, query):
|
|
try:
|
|
if query.startswith("?"):
|
|
return self._parse_simple(query)
|
|
return self._parse_jsurl(query)
|
|
except Exception as exc:
|
|
self.log.error("Invalid search query: '%s' (%s)", query, exc)
|
|
raise exception.StopExtraction()
|
|
|
|
@staticmethod
|
|
def _parse_simple(query):
|
|
"""Parse search query with format '?<key>=value>'"""
|
|
key, _, value = query.partition("=")
|
|
tag_types = {
|
|
"Tag": "1",
|
|
"Category": "2",
|
|
"Collection": "3",
|
|
"Group": "4",
|
|
"Artist": "5",
|
|
"Parody": "6",
|
|
"Character": "7",
|
|
"Uploader": "100",
|
|
}
|
|
|
|
return {
|
|
"Tags[0][Type]": tag_types[key[1:].capitalize()],
|
|
"Tags[0][Text]": text.unquote(value).replace("+", " "),
|
|
"Tags[0][Exclude]": "false",
|
|
}
|
|
|
|
@staticmethod
|
|
def _parse_jsurl(data):
|
|
"""Parse search query in JSURL format
|
|
|
|
Nested lists and dicts are handled in a special way to deal
|
|
with the way Tsumino expects its parameters -> expand(...)
|
|
|
|
Example: ~(name~'John*20Doe~age~42~children~(~'Mary~'Bill))
|
|
Ref: https://github.com/Sage/jsurl
|
|
"""
|
|
if not data:
|
|
return {}
|
|
i = 0
|
|
imax = len(data)
|
|
|
|
def eat(expected):
|
|
nonlocal i
|
|
|
|
if data[i] != expected:
|
|
error = "bad JSURL syntax: expected '{}', got {}".format(
|
|
expected, data[i])
|
|
raise ValueError(error)
|
|
i += 1
|
|
|
|
def decode():
|
|
nonlocal i
|
|
|
|
beg = i
|
|
result = ""
|
|
|
|
while i < imax:
|
|
ch = data[i]
|
|
|
|
if ch not in "~)*!":
|
|
i += 1
|
|
|
|
elif ch == "*":
|
|
if beg < i:
|
|
result += data[beg:i]
|
|
if data[i + 1] == "*":
|
|
result += chr(int(data[i+2:i+6], 16))
|
|
i += 6
|
|
else:
|
|
result += chr(int(data[i+1:i+3], 16))
|
|
i += 3
|
|
beg = i
|
|
|
|
elif ch == "!":
|
|
if beg < i:
|
|
result += data[beg:i]
|
|
result += "$"
|
|
i += 1
|
|
beg = i
|
|
|
|
else:
|
|
break
|
|
|
|
return result + data[beg:i]
|
|
|
|
def parse_one():
|
|
nonlocal i
|
|
|
|
eat('~')
|
|
result = ""
|
|
ch = data[i]
|
|
|
|
if ch == "(":
|
|
i += 1
|
|
|
|
if data[i] == "~":
|
|
result = []
|
|
if data[i+1] == ")":
|
|
i += 1
|
|
else:
|
|
result.append(parse_one())
|
|
while data[i] == "~":
|
|
result.append(parse_one())
|
|
|
|
else:
|
|
result = {}
|
|
|
|
if data[i] != ")":
|
|
while True:
|
|
key = decode()
|
|
value = parse_one()
|
|
for ekey, evalue in expand(key, value):
|
|
result[ekey] = evalue
|
|
if data[i] != "~":
|
|
break
|
|
i += 1
|
|
eat(")")
|
|
|
|
elif ch == "'":
|
|
i += 1
|
|
result = decode()
|
|
|
|
else:
|
|
beg = i
|
|
i += 1
|
|
|
|
while i < imax and data[i] not in "~)":
|
|
i += 1
|
|
|
|
sub = data[beg:i]
|
|
if ch in "0123456789-":
|
|
fval = float(sub)
|
|
ival = int(fval)
|
|
result = ival if ival == fval else fval
|
|
else:
|
|
if sub not in ("true", "false", "null"):
|
|
raise ValueError("bad value keyword: " + sub)
|
|
result = sub
|
|
|
|
return result
|
|
|
|
def expand(key, value):
|
|
if isinstance(value, list):
|
|
for index, cvalue in enumerate(value):
|
|
ckey = "{}[{}]".format(key, index)
|
|
yield from expand(ckey, cvalue)
|
|
elif isinstance(value, dict):
|
|
for ckey, cvalue in value.items():
|
|
ckey = "{}[{}]".format(key, ckey)
|
|
yield from expand(ckey, cvalue)
|
|
else:
|
|
yield key, value
|
|
|
|
return parse_one()
|