gallery-dl/gallery_dl/extractor/luscious.py

# -*- coding: utf-8 -*-

# Copyright 2016-2021 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

"""Extractors for https://members.luscious.net/"""

from .common import Extractor, Message
from .. import text, exception


class LusciousExtractor(Extractor):
    """Base class for luscious extractors"""
    category = "luscious"
    cookiedomain = ".luscious.net"
    root = "https://members.luscious.net"

    def _graphql(self, op, variables, query):
        data = {
            "id"           : 1,
            "operationName": op,
            "query"        : query,
            "variables"    : variables,
        }
        response = self.request(
            "{}/graphql/nobatch/?operationName={}".format(self.root, op),
            method="POST", json=data, fatal=False,
        )

        if response.status_code >= 400:
            self.log.debug("Server response: %s", response.text)
            raise exception.StopExtraction(
                "GraphQL query failed ('%s %s')",
                response.status_code, response.reason)

        return response.json()["data"]


class LusciousAlbumExtractor(LusciousExtractor):
    """Extractor for image albums from luscious.net"""
    subcategory = "album"
    filename_fmt = "{category}_{album[id]}_{num:>03}.{extension}"
    directory_fmt = ("{category}", "{album[id]} {album[title]}")
    archive_fmt = "{album[id]}_{id}"
    pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
               r"/(?:albums|pictures/c/[^/?#]+/album)/[^/?#]+_(\d+)")
    test = (
        ("https://luscious.net/albums/okinami-no-koigokoro_277031/", {
            "url": "7e4984a271a1072ac6483e4228a045895aff86f3",
            #  "content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3",
            "keyword": {
                "album": {
                    "__typename"  : "Album",
                    "audiences"   : list,
                    "content"     : "Hentai",
                    "cover"       : "re:https://\\w+.luscious.net/.+/277031/",
                    "created"     : 1479625853,
                    "created_by"  : "NTRshouldbeillegal",
                    "date"        : "dt:2016-11-20 07:10:53",
                    "description" : "Enjoy.",
                    "download_url": "re:/download/(r/)?824778/277031/",
                    "genres"      : list,
                    "id"          : 277031,
                    "is_manga"    : True,
                    "labels"      : list,
                    "language"    : "English",
                    "like_status" : "none",
                    "modified"    : int,
                    "permissions" : list,
                    "rating"      : float,
                    "slug"        : "okinami-no-koigokoro",
                    "status"      : str,
                    "tags"        : list,
                    "title"       : "Okinami no Koigokoro",
                    "url"         : "/albums/okinami-no-koigokoro_277031/",
                    "marked_for_deletion": False,
                    "marked_for_processing": False,
                    "number_of_animated_pictures": 0,
                    "number_of_favorites": int,
                    "number_of_pictures": 18,
                },
                "aspect_ratio": r"re:\d+:\d+",
                "category"    : "luscious",
                "created"     : int,
                "date"        : "type:datetime",
                "height"      : int,
                "id"          : int,
                "is_animated" : False,
                "like_status" : "none",
                "position"    : int,
                "resolution"  : r"re:\d+x\d+",
                "status"      : str,
                "tags"        : list,
                "thumbnail"   : str,
                "title"       : str,
                "width"       : int,
                "number_of_comments": int,
                "number_of_favorites": int,
            },
        }),
        ("https://luscious.net/albums/not-found_277035/", {
            "exception": exception.NotFoundError,
        }),
        ("https://members.luscious.net/albums/login-required_323871/", {
            "count": 78,
        }),
        ("https://www.luscious.net/albums/okinami_277031/"),
        ("https://members.luscious.net/albums/okinami_277031/"),
        ("https://luscious.net/pictures/c/video_game_manga/album"
         "/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1"),
    )

    def __init__(self, match):
        LusciousExtractor.__init__(self, match)
        self.album_id = match.group(1)
        self.gif = self.config("gif", False)

    def items(self):
        album = self.metadata()
        yield Message.Version, 1
        yield Message.Directory, {"album": album}
        for num, image in enumerate(self.images(), 1):
            image["num"] = num
            image["album"] = album

            image["thumbnail"] = image.pop("thumbnails")[0]["url"]
            image["tags"] = [item["text"] for item in image["tags"]]
            image["date"] = text.parse_timestamp(image["created"])
            image["id"] = text.parse_int(image["id"])

            url = (image["url_to_original"] or image["url_to_video"]
                   if self.gif else
                   image["url_to_video"] or image["url_to_original"])

            yield Message.Url, url, text.nameext_from_url(url, image)

    def metadata(self):
        variables = {
            "id": self.album_id,
        }

        query = """
query AlbumGet($id: ID!) {
    album {
        get(id: $id) {
            ... on Album {
                ...AlbumStandard
            }
            ... on MutationError {
                errors {
                    code
                    message
                }
            }
        }
    }
}

fragment AlbumStandard on Album {
    __typename
    id
    title
    labels
    description
    created
    modified
    like_status
    number_of_favorites
    rating
    status
    marked_for_deletion
    marked_for_processing
    number_of_pictures
    number_of_animated_pictures
    slug
    is_manga
    url
    download_url
    permissions
    cover {
        width
        height
        size
        url
    }
    created_by {
        id
        name
        display_name
        user_title
        avatar {
            url
            size
        }
        url
    }
    content {
        id
        title
        url
    }
    language {
        id
        title
        url
    }
    tags {
        id
        category
        text
        url
        count
    }
    genres {
        id
        title
        slug
        url
    }
    audiences {
        id
        title
        url
        url
    }
    last_viewed_picture {
        id
        position
        url
    }
}
"""
        album = self._graphql("AlbumGet", variables, query)["album"]["get"]
        if "errors" in album:
            raise exception.NotFoundError("album")

        album["audiences"] = [item["title"] for item in album["audiences"]]
        album["genres"] = [item["title"] for item in album["genres"]]
        album["tags"] = [item["text"] for item in album["tags"]]

        album["cover"] = album["cover"]["url"]
        album["content"] = album["content"]["title"]
        album["language"] = album["language"]["title"].partition(" ")[0]
        album["created_by"] = album["created_by"]["display_name"]

        album["id"] = text.parse_int(album["id"])
        album["date"] = text.parse_timestamp(album["created"])

        return album

    def images(self):
        variables = {
            "input": {
                "filters": [{
                    "name" : "album_id",
                    "value": self.album_id,
                }],
                "display": "position",
                "page"   : 1,
            },
        }

        query = """
query AlbumListOwnPictures($input: PictureListInput!) {
    picture {
        list(input: $input) {
            info {
                ...FacetCollectionInfo
            }
            items {
                ...PictureStandardWithoutAlbum
            }
        }
    }
}

fragment FacetCollectionInfo on FacetCollectionInfo {
    page
    has_next_page
    has_previous_page
    total_items
    total_pages
    items_per_page
    url_complete
    url_filters_only
}

fragment PictureStandardWithoutAlbum on Picture {
    __typename
    id
    title
    created
    like_status
    number_of_comments
    number_of_favorites
    status
    width
    height
    resolution
    aspect_ratio
    url_to_original
    url_to_video
    is_animated
    position
    tags {
        id
        category
        text
        url
    }
    permissions
    url
    thumbnails {
        width
        height
        size
        url
    }
}
"""
        while True:
            data = self._graphql("AlbumListOwnPictures", variables, query)
            yield from data["picture"]["list"]["items"]

            if not data["picture"]["list"]["info"]["has_next_page"]:
                return
            variables["input"]["page"] += 1


class LusciousSearchExtractor(LusciousExtractor):
    """Extractor for album searches on luscious.net"""
    subcategory = "search"
    pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
               r"/albums/list/?(?:\?([^#]+))?")
    test = (
        ("https://members.luscious.net/albums/list/"),
        ("https://members.luscious.net/albums/list/"
         "?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", {
             "pattern": LusciousAlbumExtractor.pattern,
             "range": "41-60",
             "count": 20,
         }),
    )

    def __init__(self, match):
        LusciousExtractor.__init__(self, match)
        self.query = match.group(1)

    def items(self):
        query = text.parse_query(self.query)
        display = query.pop("display", "date_newest")
        page = query.pop("page", None)

        variables = {
            "input": {
                "display": display,
                "filters": [{"name": n, "value": v} for n, v in query.items()],
                "page": text.parse_int(page, 1),
            },
        }

        query = """
query AlbumListWithPeek($input: AlbumListInput!) {
    album {
        list(input: $input) {
            info {
                ...FacetCollectionInfo
            }
            items {
                ...AlbumMinimal
                peek_thumbnails {
                    width
                    height
                    size
                    url
                }
            }
        }
    }
}

fragment FacetCollectionInfo on FacetCollectionInfo {
    page
    has_next_page
    has_previous_page
    total_items
    total_pages
    items_per_page
    url_complete
    url_filters_only
}

fragment AlbumMinimal on Album {
    __typename
    id
    title
    labels
    description
    created
    modified
    number_of_favorites
    number_of_pictures
    slug
    is_manga
    url
    download_url
    cover {
        width
        height
        size
        url
    }
    content {
        id
        title
        url
    }
    language {
        id
        title
        url
    }
    tags {
        id
        category
        text
        url
        count
    }
    genres {
        id
        title
        slug
        url
    }
    audiences {
        id
        title
        url
    }
}
"""
        yield Message.Version, 1
        while True:
            data = self._graphql("AlbumListWithPeek", variables, query)

            for album in data["album"]["list"]["items"]:
                album["url"] = self.root + album["url"]
                album["_extractor"] = LusciousAlbumExtractor
                yield Message.Queue, album["url"], album

            if not data["album"]["list"]["info"]["has_next_page"]:
                return
            variables["input"]["page"] += 1