mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-23 19:22:32 +01:00
968d3e8465
'/?&#' -> '/?#' and '?&#' -> '?#' According to https://www.ietf.org/rfc/rfc3986.txt, URLs are "organized hierarchically" by using "the slash ("/"), question mark ("?"), and number sign ("#") characters to delimit components"
457 lines
12 KiB
Python
457 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2016-2020 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Extractors for https://members.luscious.net/"""
|
|
|
|
from .common import Extractor, Message
|
|
from .. import text, exception
|
|
|
|
|
|
class LusciousExtractor(Extractor):
|
|
"""Base class for luscious extractors"""
|
|
category = "luscious"
|
|
cookiedomain = ".luscious.net"
|
|
root = "https://members.luscious.net"
|
|
|
|
def _graphql(self, op, variables, query):
|
|
data = {
|
|
"id" : 1,
|
|
"operationName": op,
|
|
"query" : query,
|
|
"variables" : variables,
|
|
}
|
|
response = self.request(
|
|
"{}/graphql/nobatch/?operationName={}".format(self.root, op),
|
|
method="POST", json=data, fatal=False,
|
|
)
|
|
|
|
if response.status_code >= 400:
|
|
self.log.debug("Server response: %s", response.text)
|
|
raise exception.StopExtraction(
|
|
"GraphQL query failed ('%s %s')",
|
|
response.status_code, response.reason)
|
|
|
|
return response.json()["data"]
|
|
|
|
|
|
class LusciousAlbumExtractor(LusciousExtractor):
|
|
"""Extractor for image albums from luscious.net"""
|
|
subcategory = "album"
|
|
filename_fmt = "{category}_{album[id]}_{num:>03}.{extension}"
|
|
directory_fmt = ("{category}", "{album[id]} {album[title]}")
|
|
archive_fmt = "{album[id]}_{id}"
|
|
pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
|
|
r"/(?:albums|pictures/c/[^/?#]+/album)/[^/?#]+_(\d+)")
|
|
test = (
|
|
("https://luscious.net/albums/okinami-no-koigokoro_277031/", {
|
|
"url": "7e4984a271a1072ac6483e4228a045895aff86f3",
|
|
# "content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3",
|
|
"keyword": {
|
|
"album": {
|
|
"__typename" : "Album",
|
|
"audiences" : list,
|
|
"content" : "Hentai",
|
|
"cover" : "re:https://\\w+.luscious.net/.+/277031/",
|
|
"created" : 1479625853,
|
|
"created_by" : "NTRshouldbeillegal",
|
|
"date" : "dt:2016-11-20 07:10:53",
|
|
"description" : "Enjoy.",
|
|
"download_url": "re:/download/(r/)?824778/277031/",
|
|
"genres" : list,
|
|
"id" : 277031,
|
|
"is_manga" : True,
|
|
"labels" : list,
|
|
"language" : "English",
|
|
"like_status" : "none",
|
|
"modified" : int,
|
|
"permissions" : list,
|
|
"rating" : float,
|
|
"slug" : "okinami-no-koigokoro",
|
|
"status" : str,
|
|
"tags" : list,
|
|
"title" : "Okinami no Koigokoro",
|
|
"url" : "/albums/okinami-no-koigokoro_277031/",
|
|
"marked_for_deletion": False,
|
|
"marked_for_processing": False,
|
|
"number_of_animated_pictures": 0,
|
|
"number_of_favorites": int,
|
|
"number_of_pictures": 18,
|
|
},
|
|
"aspect_ratio": r"re:\d+:\d+",
|
|
"category" : "luscious",
|
|
"created" : int,
|
|
"date" : "type:datetime",
|
|
"height" : int,
|
|
"id" : int,
|
|
"is_animated" : False,
|
|
"like_status" : "none",
|
|
"position" : int,
|
|
"resolution" : r"re:\d+x\d+",
|
|
"status" : str,
|
|
"tags" : list,
|
|
"thumbnail" : str,
|
|
"title" : str,
|
|
"width" : int,
|
|
"number_of_comments": int,
|
|
"number_of_favorites": int,
|
|
},
|
|
}),
|
|
("https://luscious.net/albums/virgin-killer-sweater_282582/", {
|
|
"url": "0be0cc279be1de99f727764819e03435e2a79915",
|
|
}),
|
|
("https://luscious.net/albums/not-found_277035/", {
|
|
"exception": exception.NotFoundError,
|
|
}),
|
|
("https://members.luscious.net/albums/login-required_323871/", {
|
|
"count": 78,
|
|
}),
|
|
("https://www.luscious.net/albums/okinami_277031/"),
|
|
("https://members.luscious.net/albums/okinami_277031/"),
|
|
("https://luscious.net/pictures/c/video_game_manga/album"
|
|
"/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1"),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
LusciousExtractor.__init__(self, match)
|
|
self.album_id = match.group(1)
|
|
|
|
def items(self):
|
|
album = self.metadata()
|
|
yield Message.Version, 1
|
|
yield Message.Directory, {"album": album}
|
|
for num, image in enumerate(self.images(), 1):
|
|
image["num"] = num
|
|
image["album"] = album
|
|
|
|
image["thumbnail"] = image.pop("thumbnails")[0]["url"]
|
|
image["tags"] = [item["text"] for item in image["tags"]]
|
|
image["date"] = text.parse_timestamp(image["created"])
|
|
image["id"] = text.parse_int(image["id"])
|
|
|
|
url = image["url_to_video"] or image["url_to_original"]
|
|
yield Message.Url, url, text.nameext_from_url(url, image)
|
|
|
|
def metadata(self):
|
|
variables = {
|
|
"id": self.album_id,
|
|
}
|
|
|
|
query = """
|
|
query AlbumGet($id: ID!) {
|
|
album {
|
|
get(id: $id) {
|
|
... on Album {
|
|
...AlbumStandard
|
|
}
|
|
... on MutationError {
|
|
errors {
|
|
code
|
|
message
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fragment AlbumStandard on Album {
|
|
__typename
|
|
id
|
|
title
|
|
labels
|
|
description
|
|
created
|
|
modified
|
|
like_status
|
|
number_of_favorites
|
|
rating
|
|
status
|
|
marked_for_deletion
|
|
marked_for_processing
|
|
number_of_pictures
|
|
number_of_animated_pictures
|
|
slug
|
|
is_manga
|
|
url
|
|
download_url
|
|
permissions
|
|
cover {
|
|
width
|
|
height
|
|
size
|
|
url
|
|
}
|
|
created_by {
|
|
id
|
|
name
|
|
display_name
|
|
user_title
|
|
avatar {
|
|
url
|
|
size
|
|
}
|
|
url
|
|
}
|
|
content {
|
|
id
|
|
title
|
|
url
|
|
}
|
|
language {
|
|
id
|
|
title
|
|
url
|
|
}
|
|
tags {
|
|
id
|
|
category
|
|
text
|
|
url
|
|
count
|
|
}
|
|
genres {
|
|
id
|
|
title
|
|
slug
|
|
url
|
|
}
|
|
audiences {
|
|
id
|
|
title
|
|
url
|
|
url
|
|
}
|
|
last_viewed_picture {
|
|
id
|
|
position
|
|
url
|
|
}
|
|
}
|
|
"""
|
|
album = self._graphql("AlbumGet", variables, query)["album"]["get"]
|
|
if "errors" in album:
|
|
raise exception.NotFoundError("album")
|
|
|
|
album["audiences"] = [item["title"] for item in album["audiences"]]
|
|
album["genres"] = [item["title"] for item in album["genres"]]
|
|
album["tags"] = [item["text"] for item in album["tags"]]
|
|
|
|
album["cover"] = album["cover"]["url"]
|
|
album["content"] = album["content"]["title"]
|
|
album["language"] = album["language"]["title"].partition(" ")[0]
|
|
album["created_by"] = album["created_by"]["display_name"]
|
|
|
|
album["id"] = text.parse_int(album["id"])
|
|
album["date"] = text.parse_timestamp(album["created"])
|
|
|
|
return album
|
|
|
|
def images(self):
|
|
variables = {
|
|
"input": {
|
|
"filters": [{
|
|
"name" : "album_id",
|
|
"value": self.album_id,
|
|
}],
|
|
"display": "position",
|
|
"page" : 1,
|
|
},
|
|
}
|
|
|
|
query = """
|
|
query AlbumListOwnPictures($input: PictureListInput!) {
|
|
picture {
|
|
list(input: $input) {
|
|
info {
|
|
...FacetCollectionInfo
|
|
}
|
|
items {
|
|
...PictureStandardWithoutAlbum
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fragment FacetCollectionInfo on FacetCollectionInfo {
|
|
page
|
|
has_next_page
|
|
has_previous_page
|
|
total_items
|
|
total_pages
|
|
items_per_page
|
|
url_complete
|
|
url_filters_only
|
|
}
|
|
|
|
fragment PictureStandardWithoutAlbum on Picture {
|
|
__typename
|
|
id
|
|
title
|
|
created
|
|
like_status
|
|
number_of_comments
|
|
number_of_favorites
|
|
status
|
|
width
|
|
height
|
|
resolution
|
|
aspect_ratio
|
|
url_to_original
|
|
url_to_video
|
|
is_animated
|
|
position
|
|
tags {
|
|
id
|
|
category
|
|
text
|
|
url
|
|
}
|
|
permissions
|
|
url
|
|
thumbnails {
|
|
width
|
|
height
|
|
size
|
|
url
|
|
}
|
|
}
|
|
"""
|
|
while True:
|
|
data = self._graphql("AlbumListOwnPictures", variables, query)
|
|
yield from data["picture"]["list"]["items"]
|
|
|
|
if not data["picture"]["list"]["info"]["has_next_page"]:
|
|
return
|
|
variables["input"]["page"] += 1
|
|
|
|
|
|
class LusciousSearchExtractor(LusciousExtractor):
|
|
"""Extractor for album searches on luscious.net"""
|
|
subcategory = "search"
|
|
pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
|
|
r"/albums/list/?(?:\?([^#]+))?")
|
|
test = (
|
|
("https://members.luscious.net/albums/list/"),
|
|
("https://members.luscious.net/albums/list/"
|
|
"?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", {
|
|
"pattern": LusciousAlbumExtractor.pattern,
|
|
"range": "41-60",
|
|
"count": 20,
|
|
}),
|
|
)
|
|
|
|
def __init__(self, match):
|
|
LusciousExtractor.__init__(self, match)
|
|
self.query = match.group(1)
|
|
|
|
def items(self):
|
|
query = text.parse_query(self.query)
|
|
display = query.pop("display", "date_newest")
|
|
page = query.pop("page", None)
|
|
|
|
variables = {
|
|
"input": {
|
|
"display": display,
|
|
"filters": [{"name": n, "value": v} for n, v in query.items()],
|
|
"page": text.parse_int(page, 1),
|
|
},
|
|
}
|
|
|
|
query = """
|
|
query AlbumListWithPeek($input: AlbumListInput!) {
|
|
album {
|
|
list(input: $input) {
|
|
info {
|
|
...FacetCollectionInfo
|
|
}
|
|
items {
|
|
...AlbumMinimal
|
|
peek_thumbnails {
|
|
width
|
|
height
|
|
size
|
|
url
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fragment FacetCollectionInfo on FacetCollectionInfo {
|
|
page
|
|
has_next_page
|
|
has_previous_page
|
|
total_items
|
|
total_pages
|
|
items_per_page
|
|
url_complete
|
|
url_filters_only
|
|
}
|
|
|
|
fragment AlbumMinimal on Album {
|
|
__typename
|
|
id
|
|
title
|
|
labels
|
|
description
|
|
created
|
|
modified
|
|
number_of_favorites
|
|
number_of_pictures
|
|
slug
|
|
is_manga
|
|
url
|
|
download_url
|
|
cover {
|
|
width
|
|
height
|
|
size
|
|
url
|
|
}
|
|
content {
|
|
id
|
|
title
|
|
url
|
|
}
|
|
language {
|
|
id
|
|
title
|
|
url
|
|
}
|
|
tags {
|
|
id
|
|
category
|
|
text
|
|
url
|
|
count
|
|
}
|
|
genres {
|
|
id
|
|
title
|
|
slug
|
|
url
|
|
}
|
|
audiences {
|
|
id
|
|
title
|
|
url
|
|
}
|
|
}
|
|
"""
|
|
yield Message.Version, 1
|
|
while True:
|
|
data = self._graphql("AlbumListWithPeek", variables, query)
|
|
|
|
for album in data["album"]["list"]["items"]:
|
|
album["url"] = self.root + album["url"]
|
|
album["_extractor"] = LusciousAlbumExtractor
|
|
yield Message.Queue, album["url"], album
|
|
|
|
if not data["album"]["list"]["info"]["has_next_page"]:
|
|
return
|
|
variables["input"]["page"] += 1
|