From d5fcffccedf847de2f139be362dad6cc8e415c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 17 Jul 2020 19:42:32 +0200 Subject: [PATCH] [subscribestar] add login capabilities (#852) --- README.rst | 2 +- docs/configuration.rst | 1 + docs/supportedsites.rst | 2 +- gallery_dl/extractor/subscribestar.py | 43 ++++++++++++++++++++++++++- scripts/supportedsites.py | 2 +- 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 1f9cc5f1..5bb21f4f 100644 --- a/README.rst +++ b/README.rst @@ -206,7 +206,7 @@ a username & password pair. This is necessary for ``pixiv``, ``nijie``, and ``seiga`` and optional for ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``instagram``, -``luscious``, ``sankaku``, ``tsumino``, and ``twitter``. +``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``, and ``twitter``. You can set the necessary information in your configuration file (cf. gallery-dl.conf_) diff --git a/docs/configuration.rst b/docs/configuration.rst index 661213eb..19f4809a 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -231,6 +231,7 @@ Description The username and password to use when attempting to log in to * ``instagram`` * ``luscious`` * ``sankaku`` + * ``subscribestar`` * ``tsumino`` * ``twitter`` diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 2ed5d6ec..721b0f4e 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -121,7 +121,7 @@ SlickPic https://www.slickpic.com/ Albums, User Profiles SlideShare https://www.slideshare.net/ Presentations SmugMug https://www.smugmug.com/ |smugmug-C| Optional (`OAuth `__) Speaker Deck https://speakerdeck.com/ Presentations -SubscribeStar https://www.subscribestar.com/ Posts, User Profiles Optional (`Cookies `__) +SubscribeStar https://www.subscribestar.com/ Posts, User Profiles Optional The /b/ Archive https://thebarchive.com/ Threads Tsumino https://www.tsumino.com/ Galleries, Search Results Optional Tumblr https://www.tumblr.com/ Likes, Posts, Tag Searches, User Profiles Optional (`OAuth `__) diff --git a/gallery_dl/extractor/subscribestar.py b/gallery_dl/extractor/subscribestar.py index 07fffcbd..08d88509 100644 --- a/gallery_dl/extractor/subscribestar.py +++ b/gallery_dl/extractor/subscribestar.py @@ -9,7 +9,8 @@ """Extractors for https://www.subscribestar.com/""" from .common import Extractor, Message -from .. import text +from .. import text, exception +from ..cache import cache import datetime import json @@ -24,17 +25,21 @@ class SubscribestarExtractor(Extractor): directory_fmt = ("{category}", "{author_name}") filename_fmt = "{post_id}_{id}.{extension}" archive_fmt = "{id}" + cookiedomain = "www.subscribestar.com" + cookienames = ("auth_token",) def __init__(self, match): tld, self.item = match.groups() if tld == "adult": self.root = "https://subscribestar.adult" + self.cookiedomain = "subscribestar.adult" self.subcategory += "-adult" Extractor.__init__(self, match) self.metadata = self.config("metadata", False) self._year = " " + str(datetime.date.today().year) def items(self): + self.login() for post_html in self.posts(): media = self._media_from_post(post_html) if not media: @@ -49,6 +54,42 @@ class SubscribestarExtractor(Extractor): def posts(self): """Yield HTML content of all relevant posts""" + def login(self): + if self._check_cookies(self.cookienames): + return + username, password = self._get_auth_info() + if username: + cookies = self._login_impl(username, password) + self._update_cookies(cookies) + + @cache(maxage=28*24*3600, keyarg=1) + def _login_impl(self, username, password): + self.log.info("Logging in as %s", username) + + url = "https://www.subscribestar.com/session.json" + headers = { + "Origin" : "https://www.subscribestar.com", + "Referer" : "https://www.subscribestar.com/login", + "X-Requested-With": "XMLHttpRequest", + } + data = { + "utf8" : "✓", + "email" : username, + "password": password, + } + + response = self.request( + url, method="POST", headers=headers, data=data, fatal=False) + if response.json().get("errors"): + self.log.debug(response.json()["errors"]) + raise exception.AuthenticationError() + + return { + cookie.name: cookie.value + for cookie in response.cookies + if cookie.name.startswith("auth") + } + @staticmethod def _media_from_post(html): gallery = text.extract(html, 'data-gallery="', '"')[0] diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 628436b1..80e6ed7d 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -172,7 +172,7 @@ AUTH_MAP = { "sankaku" : "Optional", "seiga" : "Required", "smugmug" : "Optional" + _OAUTH, - "subscribestar" : "Optional" + _COOKIES, + "subscribestar" : "Optional", "tsumino" : "Optional", "tumblr" : "Optional" + _OAUTH, "twitter" : "Optional",