Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-25 04:02:32 +01:00
Code Issues Releases Wiki Activity

[pixiv] update (#1304)

Browse Source 
- remove login with username & password
- require a refresh token
- add 'oauth:pixiv' functionality

See also:
- https://github.com/upbit/pixivpy/issues/158
- https://gist.github.com/ZipFile/c9ebedb224406f4f11845ab700124362
This commit is contained in:
Mike Fährmann 2021-02-12 18:03:06 +01:00
parent 79c0fc249b
commit 8974f0361c
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
6 changed files with 99 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.rst
View File

@ -214,7 +214,7 @@ Username & Password
Some extractors require you to provide valid login credentials in the form of
a username & password pair. This is necessary for
``pixiv``, ``nijie``, and ``seiga``
``nijie`` and ``seiga``
and optional for
``aryion``,
``danbooru``,
@ -237,7 +237,7 @@ You can set the necessary information in your configuration file
{
"extractor": {
"pixiv": {
"seiga": {
"username": "<username>",
"password": "<password>"
}

1
docs/configuration.rst
View File

@ -272,7 +272,6 @@ Description
Specifying a username and password is required for
* ``pixiv``
* ``nijie``
* ``seiga``

2
docs/supportedsites.rst
View File

@ -103,7 +103,7 @@ PhotoVogue https://www.vogue.it/en/photovogue/ User Profiles
Piczel https://piczel.tv/ Folders, individual Images, User Profiles
Pillowfort https://www.pillowfort.social/ Posts, User Profiles
Pinterest https://www.pinterest.com/ |pinterest-C| Supported
Pixiv https://www.pixiv.net/ |pixiv-C| Required
Pixiv https://www.pixiv.net/ |pixiv-C| `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
Pixnet https://www.pixnet.net/ Folders, individual Images, Sets, User Profiles
Plurk https://www.plurk.com/ Posts, Timelines
Pornhub https://www.pornhub.com/ Galleries, User Profiles

73
gallery_dl/extractor/oauth.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017-2020 Mike Fährmann
# Copyright 2017-2021 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@ -9,10 +9,12 @@
"""Utility classes to setup OAuth and link accounts to gallery-dl"""
from .common import Extractor, Message
from . import deviantart, flickr, reddit, smugmug, tumblr
from . import deviantart, flickr, pixiv, reddit, smugmug, tumblr
from .. import text, oauth, util, config, exception
from ..cache import cache
import urllib.parse
import hashlib
import base64
REDIRECT_URI_LOCALHOST = "http://localhost:6414/"
REDIRECT_URI_HTTPS = "https://mikf.github.io/gallery-dl/oauth-redirect.html"
@ -62,14 +64,14 @@ class OAuthBase(Extractor):
self.client.send(b"HTTP/1.1 200 OK\r\n\r\n" + msg.encode())
self.client.close()
def open(self, url, params):
def open(self, url, params, recv=None):
"""Open 'url' in browser amd return response parameters"""
import webbrowser
url += "?" + urllib.parse.urlencode(params)
if not self.config("browser", True) or not webbrowser.open(url):
print("Please open this URL in your browser:")
print(url, end="\n\n", flush=True)
return self.recv()
return (recv or self.recv)()
def _oauth1_authorization_flow(
self, request_token_url, authorize_url, access_token_url):
@ -362,6 +364,69 @@ class OAuthMastodon(OAuthBase):
return data
class OAuthPixiv(OAuthBase):
subcategory = "pixiv"
pattern = "oauth:pixiv$"
def items(self):
yield Message.Version, 1
code_verifier = util.generate_token(32)
digest = hashlib.sha256(code_verifier.encode("ascii")).digest()
code_challenge = base64.urlsafe_b64encode(
digest).rstrip(b"=").decode("ascii")
url = "https://app-api.pixiv.net/web/v1/login"
params = {
"code_challenge": code_challenge,
"code_challenge_method": "S256",
"client": "pixiv-android",
}
code = self.open(url, params, self._input)
url = "https://oauth.secure.pixiv.net/auth/token"
headers = {
"User-Agent": "PixivAndroidApp/5.0.234 (Android 11; Pixel 5)",
}
data = {
"client_id" : self.oauth_config(
"client-id" , pixiv.PixivAppAPI.CLIENT_ID),
"client_secret" : self.oauth_config(
"client-secret", pixiv.PixivAppAPI.CLIENT_SECRET),
"code" : code,
"code_verifier" : code_verifier,
"grant_type" : "authorization_code",
"include_policy": "true",
"redirect_uri" : "https://app-api.pixiv.net"
"/web/v1/users/auth/pixiv/callback",
}
data = self.session.post(url, headers=headers, data=data).json()
if "error" in data:
print(data)
if data["error"] == "invalid_request":
print("'code' expired, try again")
return
token = data["refresh_token"]
if self.cache:
username = self.oauth_config("username")
pixiv._refresh_token_cache.update(username, token)
self.log.info("Writing 'refresh-token' to cache")
print(self._generate_message(("refresh-token",), (token,)))
def _input(self):
print("""
1) Open your browser's Developer Tools (F12) and switch to the Network tab
2) Login
4) Select the last network monitor entry ('callback?state=...')
4) Copy its 'code' query parameter, paste it below, and press Enter
""")
code = input("code: ")
return code.rpartition("=")[2].strip()
MASTODON_MSG_TEMPLATE = """
Your 'access-token' is

51
gallery_dl/extractor/pixiv.py
View File

@ -510,49 +510,48 @@ class PixivAppAPI():
def __init__(self, extractor):
self.extractor = extractor
self.log = extractor.log
self.username, self.password = extractor._get_auth_info()
self.username = extractor._get_auth_info()[0]
self.user = None
extractor.session.headers.update({
"App-OS" : "ios",
"App-OS-Version": "13.1.2",
"App-Version" : "7.7.6",
"User-Agent" : "PixivIOSApp/7.7.6 (iOS 13.1.2; iPhone11,8)",
"Referer" : "https://app-api.pixiv.net/",
})
self.client_id = extractor.config(
"client-id", self.CLIENT_ID)
self.client_secret = extractor.config(
"client-secret", self.CLIENT_SECRET)
extractor.session.headers.update({
"App-OS": "ios",
"App-OS-Version": "10.3.1",
"App-Version": "6.7.1",
"User-Agent": "PixivIOSApp/6.7.1 (iOS 10.3.1; iPhone8,1)",
"Referer": "https://app-api.pixiv.net/",
})
token = extractor.config("refresh-token")
if token is None or token == "cache":
token = _refresh_token_cache(self.username)
self.refresh_token = token
def login(self):
"""Login and gain an access token"""
self.user, auth = self._login_impl(self.username, self.password)
self.user, auth = self._login_impl(self.username)
self.extractor.session.headers["Authorization"] = auth
@cache(maxage=3600, keyarg=1)
def _login_impl(self, username, password):
if not username or not password:
def _login_impl(self, username):
if not self.refresh_token:
raise exception.AuthenticationError(
"Username and password required")
"'refresh-token' required.\n"
"Run `gallery-dl oauth:pixiv` to get one.")
self.log.info("Refreshing access token")
url = "https://oauth.secure.pixiv.net/auth/token"
data = {
"client_id" : self.client_id,
"client_secret" : self.client_secret,
"get_secure_url": 1,
"grant_type" : "refresh_token",
"refresh_token" : self.refresh_token,
"get_secure_url": "1",
}
refresh_token = _refresh_token_cache(username)
if refresh_token:
self.log.info("Refreshing access token")
data["grant_type"] = "refresh_token"
data["refresh_token"] = refresh_token
else:
self.log.info("Logging in as %s", username)
data["grant_type"] = "password"
data["username"] = username
data["password"] = password
time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00")
headers = {
@ -565,11 +564,9 @@ class PixivAppAPI():
url, method="POST", headers=headers, data=data, fatal=False)
if response.status_code >= 400:
self.log.debug(response.text)
raise exception.AuthenticationError()
raise exception.AuthenticationError("Invalid refresh token")
data = response.json()["response"]
if not refresh_token:
_refresh_token_cache.update(username, data["refresh_token"])
return data["user"], "Bearer " + data["access_token"]
def illust_detail(self, illust_id):

2
scripts/supportedsites.py
View File

@ -189,7 +189,7 @@ AUTH_MAP = {
"patreon" : _COOKIES,
"pawoo" : _OAUTH,
"pinterest" : "Supported",
"pixiv" : "Required",
"pixiv" : _OAUTH,
"reddit" : _OAUTH,
"sankaku" : "Supported",
"seiga" : "Required",