gallery-dl/gallery_dl/extractor/luscious.py

# -*- coding: utf-8 -*-

# Copyright 2016-2019 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

"""Extractors for https://members.luscious.net/"""

from .common import Extractor, Message
from .. import text, exception


class LusciousExtractor(Extractor):
    """Base class for luscious extractors"""
    category = "luscious"
    cookiedomain = ".luscious.net"
    root = "https://members.luscious.net"

    def _graphql(self, op, variables, query):
        data = {
            "id"           : 1,
            "operationName": op,
            "query"        : query,
            "variables"    : variables,
        }
        response = self.request(
            "{}/graphql/nobatch/?operationName={}".format(self.root, op),
            method="POST", json=data, fatal=False,
        )

        if response.status_code >= 400:
            self.log.error("GraphQL query failed ('%s %s')",
                           response.status_code, response.reason)
            self.log.debug("Server response: %s", response.text)
            raise exception.StopExtraction()

        return response.json()["data"]


class LusciousAlbumExtractor(LusciousExtractor):
    """Extractor for image albums from luscious.net"""
    subcategory = "album"
    filename_fmt = "{category}_{album[id]}_{num:>03}.{extension}"
    directory_fmt = ("{category}", "{album[id]} {album[title]}")
    archive_fmt = "{album[id]}_{id}"
    pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
               r"/(?:albums|pictures/c/[^/?&#]+/album)/[^/?&#]+_(\d+)")
    test = (
        ("https://luscious.net/albums/okinami-no-koigokoro_277031/", {
            "url": "7e4984a271a1072ac6483e4228a045895aff86f3",
            #  "keyword": "07c0b915f2ab1cc3bbf28b76e7950fccee1213f3",
            #  "content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3",
        }),
        ("https://luscious.net/albums/virgin-killer-sweater_282582/", {
            "url": "21cc68a7548f4d71dfd67d8caf96349dde7e791c",
            #  "keyword": "e1202078b504adeccd521aa932f456a5a85479a0",
        }),
        ("https://luscious.net/albums/not-found_277035/", {
            "exception": exception.NotFoundError,
        }),
        ("https://members.luscious.net/albums/login-required_323871/", {
            "count": 78,
        }),
        ("https://www.luscious.net/albums/okinami_277031/"),
        ("https://members.luscious.net/albums/okinami_277031/"),
        ("https://luscious.net/pictures/c/video_game_manga/album"
         "/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1"),
    )

    def __init__(self, match):
        LusciousExtractor.__init__(self, match)
        self.album_id = match.group(1)

    def items(self):
        album = self.metadata()
        yield Message.Version, 1
        yield Message.Directory, {"album": album}
        for num, image in enumerate(self.images(), 1):
            image["num"] = num
            image["album"] = album

            image["thumbnail"] = image.pop("thumbnails")[0]["url"]
            image["tags"] = [item["text"] for item in image["tags"]]
            image["date"] = text.parse_timestamp(image["created"])
            image["id"] = text.parse_int(image["id"])

            url = image["url_to_video"] or image["url_to_original"]
            yield Message.Url, url, text.nameext_from_url(url, image)

    def metadata(self):
        variables = {
            "id": self.album_id,
        }
        query = (
            "query AlbumGet($id: ID!) { album { get(id: $id) { ... on Album { "
            "...AlbumStandard } ... on MutationError { errors { code message }"
            " } } } } fragment AlbumStandard on Album { __typename id title la"
            "bels description created modified like_status number_of_favorites"
            " rating status marked_for_deletion marked_for_processing number_o"
            "f_pictures number_of_animated_pictures slug is_manga url download"
            "_url permissions cover { width height size url } created_by { id "
            "name display_name user_title name display_name avatar { url size "
            "} url } content { id title url } language { id title url } tags {"
            " id category slug text url count } genres { id title slug url } a"
            "udiences { id title url url } last_viewed_picture { id position u"
            "rl } }"
        )

        album = self._graphql("AlbumGet", variables, query)["album"]["get"]
        if "errors" in album:
            raise exception.NotFoundError("album")

        album["audiences"] = [item["title"] for item in album["audiences"]]
        album["genres"] = [item["title"] for item in album["genres"]]
        album["tags"] = [item["text"] for item in album["tags"]]

        album["cover"] = album["cover"]["url"]
        album["content"] = album["content"]["title"]
        album["language"] = album["language"]["title"].partition(" ")[0]
        album["created_by"] = album["created_by"]["display_name"]

        album["id"] = text.parse_int(album["id"])
        album["date"] = text.parse_timestamp(album["created"])

        return album

    def images(self):
        variables = {
            "input": {
                "filters": [{
                    "name" : "album_id",
                    "value": self.album_id,
                }],
                "display": "position",
                "page"   : 1,
            },
        }
        query = (
            "query AlbumListOwnPictures($input: PictureListInput!) { picture {"
            " list(input: $input) { info { ...FacetCollectionInfo } items { .."
            ".PictureStandardWithoutAlbum } } } } fragment FacetCollectionInfo"
            " on FacetCollectionInfo { page has_next_page has_previous_page to"
            "tal_items total_pages items_per_page url_complete url_filters_onl"
            "y } fragment PictureStandardWithoutAlbum on Picture { __typename "
            "id title created like_status number_of_comments number_of_favorit"
            "es status width height resolution aspect_ratio url_to_original ur"
            "l_to_video is_animated position tags { id category slug text url "
            "} permissions url thumbnails { width height size url } }"
        )

        while True:
            data = self._graphql("AlbumListOwnPictures", variables, query)
            yield from data["picture"]["list"]["items"]

            if not data["picture"]["list"]["info"]["has_next_page"]:
                return
            variables["input"]["page"] += 1


class LusciousSearchExtractor(LusciousExtractor):
    """Extractor for album searches on luscious.net"""
    subcategory = "search"
    pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
               r"/albums/list/?(?:\?([^#]+))?")
    test = (
        ("https://members.luscious.net/albums/list/"),
        ("https://members.luscious.net/albums/list/"
         "?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", {
             "pattern": LusciousAlbumExtractor.pattern,
             "range": "41-60",
             "count": 20,
         }),
    )

    def __init__(self, match):
        LusciousExtractor.__init__(self, match)
        self.query = match.group(1)

    def items(self):
        query = text.parse_query(self.query)
        display = query.pop("display", "date_newest")
        page = query.pop("page", None)

        variables = {
            "input": {
                "display": display,
                "filters": [{"name": n, "value": v} for n, v in query.items()],
                "page": text.parse_int(page, 1),
            },
        }
        query = (
            "query AlbumListWithPeek($input: AlbumListInput!) { album { list(i"
            "nput: $input) { info { ...FacetCollectionInfo } items { ...AlbumM"
            "inimal peek_thumbnails { width height size url } } } } } fragment"
            " FacetCollectionInfo on FacetCollectionInfo { page has_next_page "
            "has_previous_page total_items total_pages items_per_page url_comp"
            "lete url_filters_only } fragment AlbumMinimal on Album { __typena"
            "me id title labels description created modified number_of_favorit"
            "es number_of_pictures slug is_manga url download_url cover { widt"
            "h height size url } content { id title url } language { id title "
            "url } tags { id category slug text url count } genres { id title "
            "slug url } audiences { id title url } }"
        )

        yield Message.Version, 1
        while True:
            data = self._graphql("AlbumListWithPeek", variables, query)

            for album in data["album"]["list"]["items"]:
                album["url"] = self.root + album["url"]
                album["_extractor"] = LusciousAlbumExtractor
                yield Message.Queue, album["url"], album

            if not data["album"]["list"]["info"]["has_next_page"]:
                return
            variables["input"]["page"] += 1