gallery-dl/gallery_dl/extractor/8muses.py

# -*- coding: utf-8 -*-

# Copyright 2019 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

"""Extractors for https://www.8muses.com/"""

from .common import Extractor, Message
from .. import text
import json


class _8musesAlbumExtractor(Extractor):
    """Extractor for image albums on www.8muses.com"""
    category = "8muses"
    subcategory = "album"
    directory_fmt = ("{category}", "{album[path]}")
    filename_fmt = "{page:>03}.{extension}"
    archive_fmt = "{hash}"
    root = "https://www.8muses.com"
    pattern = (r"(?:https?://)?(?:www\.)?8muses\.com"
               r"(/comics/album/[^?&#]+)(\?[^#]+)?")
    test = (
        ("https://www.8muses.com/comics/album/Fakku-Comics/santa/Im-Sorry", {
            "url": "82449d6a26a29204695cba5d52c3ec60170bc159",
            "keyword": {
                "url"  : str,
                "hash" : str,
                "page" : int,
                "count": 16,
                "album": {
                    "id"     : 10457,
                    "title"  : "Im Sorry",
                    "path"   : "Fakku Comics/santa/Im Sorry",
                    "private": False,
                    "url"    : str,
                    "parent" : 10454,
                    "views"  : int,
                    "likes"  : int,
                    "date"   : "type:datetime",
                },
            },
        }),
        ("https://www.8muses.com/comics/album/Fakku-Comics/santa", {
            "count": ">= 3",
            "pattern": pattern,
            "keyword": {
                "url"    : str,
                "name"   : str,
                "private": False,
            },
        }),
        ("https://www.8muses.com/comics/album/Fakku-Comics/7?sort=az", {
            "count": ">= 70",
            "keyword": {"name": r"re:^[R-Zr-z]"},
        }),
    )

    def __init__(self, match):
        Extractor.__init__(self, match)
        self.path = match.group(1)
        self.params = match.group(2) or ""

    def items(self):
        url = self.root + self.path + self.params

        while True:
            data = self._unobfuscate(text.extract(
                self.request(url).text,
                'id="ractive-public" type="text/plain">', '</script>')[0])

            images = data.get("pictures")
            if images:
                count = len(images)
                album = self._make_album(data["album"])
                yield Message.Directory, {"album": album, "count": count}
                for num, image in enumerate(images, 1):
                    url = self.root + "/image/fl/" + image["publicUri"]
                    img = {
                        "url"      : url,
                        "page"     : num,
                        "hash"     : image["publicUri"],
                        "count"    : count,
                        "album"    : album,
                        "extension": "jpg",
                    }
                    yield Message.Url, url, img

            albums = data.get("albums")
            if albums:
                for album in albums:
                    url = self.root + "/comics/album/" + album["permalink"]
                    album = {
                        "url"    : url,
                        "name"   : album["name"],
                        "private": album["isPrivate"],
                    }
                    yield Message.Queue, url, album

            if data["page"] >= data["pages"]:
                return
            path, _, num = self.path.rstrip("/").rpartition("/")
            path = path if num.isdecimal() else self.path
            url = "{}{}/{}{}".format(
                self.root, path, data["page"] + 1, self.params)

    def _make_album(self, album):
        return {
            "id"     : album["id"],
            "path"   : album["path"],
            "title"  : album["name"],
            "private": album["isPrivate"],
            "url"    : self.root + album["permalink"],
            "parent" : text.parse_int(album["parentId"]),
            "views"  : text.parse_int(album["numberViews"]),
            "likes"  : text.parse_int(album["numberLikes"]),
            "date"   : text.parse_datetime(
                album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"),
        }

    @staticmethod
    def _unobfuscate(data):
        return json.loads("".join([
            chr(33 + (ord(c) + 14) % 94) if c != " " else c
            for c in text.unescape(data.strip("\t\n\r !"))
        ]))