From 8974f0361c9cd9b53a7b7b21c96bd21b997dd596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 12 Feb 2021 18:03:06 +0100 Subject: [PATCH] [pixiv] update (#1304) - remove login with username & password - require a refresh token - add 'oauth:pixiv' functionality See also: - https://github.com/upbit/pixivpy/issues/158 - https://gist.github.com/ZipFile/c9ebedb224406f4f11845ab700124362 --- README.rst | 4 +- docs/configuration.rst | 1 - docs/supportedsites.rst | 2 +- gallery_dl/extractor/oauth.py | 73 +++++++++++++++++++++++++++++++++-- gallery_dl/extractor/pixiv.py | 55 +++++++++++++------------- scripts/supportedsites.py | 2 +- 6 files changed, 99 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index 1ddebcfe..f9dddcb7 100644 --- a/README.rst +++ b/README.rst @@ -214,7 +214,7 @@ Username & Password Some extractors require you to provide valid login credentials in the form of a username & password pair. This is necessary for -``pixiv``, ``nijie``, and ``seiga`` +``nijie`` and ``seiga`` and optional for ``aryion``, ``danbooru``, @@ -237,7 +237,7 @@ You can set the necessary information in your configuration file { "extractor": { - "pixiv": { + "seiga": { "username": "", "password": "" } diff --git a/docs/configuration.rst b/docs/configuration.rst index 7f084d56..6aa9ff88 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -272,7 +272,6 @@ Description Specifying a username and password is required for - * ``pixiv`` * ``nijie`` * ``seiga`` diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 7d993504..c2700ef1 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -103,7 +103,7 @@ PhotoVogue https://www.vogue.it/en/photovogue/ User Profiles Piczel https://piczel.tv/ Folders, individual Images, User Profiles Pillowfort https://www.pillowfort.social/ Posts, User Profiles Pinterest https://www.pinterest.com/ |pinterest-C| Supported -Pixiv https://www.pixiv.net/ |pixiv-C| Required +Pixiv https://www.pixiv.net/ |pixiv-C| `OAuth `__ Pixnet https://www.pixnet.net/ Folders, individual Images, Sets, User Profiles Plurk https://www.plurk.com/ Posts, Timelines Pornhub https://www.pornhub.com/ Galleries, User Profiles diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py index 4bb2c482..2ec71655 100644 --- a/gallery_dl/extractor/oauth.py +++ b/gallery_dl/extractor/oauth.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2020 Mike Fährmann +# Copyright 2017-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,10 +9,12 @@ """Utility classes to setup OAuth and link accounts to gallery-dl""" from .common import Extractor, Message -from . import deviantart, flickr, reddit, smugmug, tumblr +from . import deviantart, flickr, pixiv, reddit, smugmug, tumblr from .. import text, oauth, util, config, exception from ..cache import cache import urllib.parse +import hashlib +import base64 REDIRECT_URI_LOCALHOST = "http://localhost:6414/" REDIRECT_URI_HTTPS = "https://mikf.github.io/gallery-dl/oauth-redirect.html" @@ -62,14 +64,14 @@ class OAuthBase(Extractor): self.client.send(b"HTTP/1.1 200 OK\r\n\r\n" + msg.encode()) self.client.close() - def open(self, url, params): + def open(self, url, params, recv=None): """Open 'url' in browser amd return response parameters""" import webbrowser url += "?" + urllib.parse.urlencode(params) if not self.config("browser", True) or not webbrowser.open(url): print("Please open this URL in your browser:") print(url, end="\n\n", flush=True) - return self.recv() + return (recv or self.recv)() def _oauth1_authorization_flow( self, request_token_url, authorize_url, access_token_url): @@ -362,6 +364,69 @@ class OAuthMastodon(OAuthBase): return data +class OAuthPixiv(OAuthBase): + subcategory = "pixiv" + pattern = "oauth:pixiv$" + + def items(self): + yield Message.Version, 1 + + code_verifier = util.generate_token(32) + digest = hashlib.sha256(code_verifier.encode("ascii")).digest() + code_challenge = base64.urlsafe_b64encode( + digest).rstrip(b"=").decode("ascii") + + url = "https://app-api.pixiv.net/web/v1/login" + params = { + "code_challenge": code_challenge, + "code_challenge_method": "S256", + "client": "pixiv-android", + } + code = self.open(url, params, self._input) + + url = "https://oauth.secure.pixiv.net/auth/token" + headers = { + "User-Agent": "PixivAndroidApp/5.0.234 (Android 11; Pixel 5)", + } + data = { + "client_id" : self.oauth_config( + "client-id" , pixiv.PixivAppAPI.CLIENT_ID), + "client_secret" : self.oauth_config( + "client-secret", pixiv.PixivAppAPI.CLIENT_SECRET), + "code" : code, + "code_verifier" : code_verifier, + "grant_type" : "authorization_code", + "include_policy": "true", + "redirect_uri" : "https://app-api.pixiv.net" + "/web/v1/users/auth/pixiv/callback", + } + data = self.session.post(url, headers=headers, data=data).json() + + if "error" in data: + print(data) + if data["error"] == "invalid_request": + print("'code' expired, try again") + return + + token = data["refresh_token"] + if self.cache: + username = self.oauth_config("username") + pixiv._refresh_token_cache.update(username, token) + self.log.info("Writing 'refresh-token' to cache") + + print(self._generate_message(("refresh-token",), (token,))) + + def _input(self): + print(""" +1) Open your browser's Developer Tools (F12) and switch to the Network tab +2) Login +4) Select the last network monitor entry ('callback?state=...') +4) Copy its 'code' query parameter, paste it below, and press Enter +""") + code = input("code: ") + return code.rpartition("=")[2].strip() + + MASTODON_MSG_TEMPLATE = """ Your 'access-token' is diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index a872ada0..be976e96 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -510,49 +510,48 @@ class PixivAppAPI(): def __init__(self, extractor): self.extractor = extractor self.log = extractor.log - self.username, self.password = extractor._get_auth_info() + self.username = extractor._get_auth_info()[0] self.user = None + extractor.session.headers.update({ + "App-OS" : "ios", + "App-OS-Version": "13.1.2", + "App-Version" : "7.7.6", + "User-Agent" : "PixivIOSApp/7.7.6 (iOS 13.1.2; iPhone11,8)", + "Referer" : "https://app-api.pixiv.net/", + }) + self.client_id = extractor.config( "client-id", self.CLIENT_ID) self.client_secret = extractor.config( "client-secret", self.CLIENT_SECRET) - extractor.session.headers.update({ - "App-OS": "ios", - "App-OS-Version": "10.3.1", - "App-Version": "6.7.1", - "User-Agent": "PixivIOSApp/6.7.1 (iOS 10.3.1; iPhone8,1)", - "Referer": "https://app-api.pixiv.net/", - }) + + token = extractor.config("refresh-token") + if token is None or token == "cache": + token = _refresh_token_cache(self.username) + self.refresh_token = token def login(self): """Login and gain an access token""" - self.user, auth = self._login_impl(self.username, self.password) + self.user, auth = self._login_impl(self.username) self.extractor.session.headers["Authorization"] = auth @cache(maxage=3600, keyarg=1) - def _login_impl(self, username, password): - if not username or not password: + def _login_impl(self, username): + if not self.refresh_token: raise exception.AuthenticationError( - "Username and password required") + "'refresh-token' required.\n" + "Run `gallery-dl oauth:pixiv` to get one.") + self.log.info("Refreshing access token") url = "https://oauth.secure.pixiv.net/auth/token" data = { - "client_id": self.client_id, - "client_secret": self.client_secret, - "get_secure_url": 1, + "client_id" : self.client_id, + "client_secret" : self.client_secret, + "grant_type" : "refresh_token", + "refresh_token" : self.refresh_token, + "get_secure_url": "1", } - refresh_token = _refresh_token_cache(username) - - if refresh_token: - self.log.info("Refreshing access token") - data["grant_type"] = "refresh_token" - data["refresh_token"] = refresh_token - else: - self.log.info("Logging in as %s", username) - data["grant_type"] = "password" - data["username"] = username - data["password"] = password time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00") headers = { @@ -565,11 +564,9 @@ class PixivAppAPI(): url, method="POST", headers=headers, data=data, fatal=False) if response.status_code >= 400: self.log.debug(response.text) - raise exception.AuthenticationError() + raise exception.AuthenticationError("Invalid refresh token") data = response.json()["response"] - if not refresh_token: - _refresh_token_cache.update(username, data["refresh_token"]) return data["user"], "Bearer " + data["access_token"] def illust_detail(self, illust_id): diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index b3f99ad8..3a37095a 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -189,7 +189,7 @@ AUTH_MAP = { "patreon" : _COOKIES, "pawoo" : _OAUTH, "pinterest" : "Supported", - "pixiv" : "Required", + "pixiv" : _OAUTH, "reddit" : _OAUTH, "sankaku" : "Supported", "seiga" : "Required",