From 29d75fc3faeff60e1bb4403e2f4cd6de641faa6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 11 Jan 2018 14:11:37 +0100 Subject: [PATCH] [tumblr] add support for OAuth authentication (#65) --- README.rst | 2 +- docs/supportedsites.rst | 2 +- gallery_dl/extractor/oauth.py | 153 +++++++++++++++++++------------- gallery_dl/extractor/tumblr.py | 23 +++-- gallery_dl/util.py | 21 +++-- scripts/build_supportedsites.py | 1 + 6 files changed, 126 insertions(+), 76 deletions(-) diff --git a/README.rst b/README.rst index 09fa5978..70398f2c 100644 --- a/README.rst +++ b/README.rst @@ -191,7 +191,7 @@ OAuth ----- *gallery-dl* supports user authentication via OAuth_ for -``deviantart``, ``flickr`` and ``reddit``. +``deviantart``, ``flickr``, ``reddit`` and ``tumblr``. This is entirely optional, but grants *gallery-dl* the ability to issue requests on your account's behalf and enables it to access resources which would otherwise be unavailable to a public user. diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index ea3e1b47..3c5a5e8b 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -71,7 +71,7 @@ Sense-Scans http://sensescans.com/ Chapters, Manga SlideShare https://www.slideshare.net/ Presentations Spectrum Nexus |http://www.thes-0| Chapters, Manga The /b/ Archive https://thebarchive.com/ Threads -Tumblr https://www.tumblr.com/ Images from Users, Posts, Tag-Searches +Tumblr https://www.tumblr.com/ Images from Users, Posts, Tag-Searches Optional (OAuth) Twitter https://twitter.com/ Tweets Warosu https://warosu.org/ Threads World Three http://www.slide.world-three.org/ Chapters, Manga diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py index 7e816a86..bdffc259 100644 --- a/gallery_dl/extractor/oauth.py +++ b/gallery_dl/extractor/oauth.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017 Mike Fährmann +# Copyright 2017-2018 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,8 @@ """Utility classes to setup OAuth and link a users account to gallery-dl""" from .common import Extractor, Message -from . import deviantart, flickr, reddit -from .. import util +from . import deviantart, flickr, reddit, tumblr +from .. import text, util import os import urllib.parse @@ -71,6 +71,32 @@ class OAuthBase(Extractor): print(url, end="\n\n", flush=True) return self.recv() + def _oauth1_authorization_flow( + self, request_token_url, authorize_url, access_token_url): + """Perform the OAuth 1.0a authorization flow""" + del self.session.params["oauth_token"] + + # Get a Request Token + params = {"oauth_callback": self.redirect_uri} + data = self.session.get(request_token_url, params=params).text + + data = text.parse_query(data) + self.session.params["oauth_token"] = token = data["oauth_token"] + self.session.token_secret = data["oauth_token_secret"] + + # Get the User's Authorization + params = {"oauth_token": token, "perms": "read"} + data = self.open(authorize_url, params) + + # Exchange the Request Token for an Access Token + data = self.session.get(access_token_url, params=data).text + + data = text.parse_query(data) + self.send(OAUTH1_MSG_TEMPLATE.format( + category=self.subcategory, + token=data["oauth_token"], + token_secret=data["oauth_token_secret"])) + def _oauth2_authorization_code_grant( self, client_id, client_secret, auth_url, token_url, scope): """Perform an OAuth2 authorization code grant""" @@ -120,23 +146,6 @@ class OAuthBase(Extractor): )) -class OAuthReddit(OAuthBase): - subcategory = "reddit" - pattern = ["oauth:reddit$"] - - def items(self): - yield Message.Version, 1 - - self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT - self._oauth2_authorization_code_grant( - reddit.RedditAPI.CLIENT_ID, - "", - "https://www.reddit.com/api/v1/authorize", - "https://www.reddit.com/api/v1/access_token", - "read", - ) - - class OAuthDeviantart(OAuthBase): subcategory = "deviantart" pattern = ["oauth:deviantart$"] @@ -162,35 +171,79 @@ class OAuthFlickr(OAuthBase): OAuthBase.__init__(self, match) self.session = util.OAuthSession( self.session, - flickr.FlickrAPI.API_KEY, flickr.FlickrAPI.API_SECRET + flickr.FlickrAPI.API_KEY, + flickr.FlickrAPI.API_SECRET, ) - del self.session.params["oauth_token"] def items(self): yield Message.Version, 1 - # Get a Request Token - url = "https://www.flickr.com/services/oauth/request_token" - params = {"oauth_callback": self.redirect_uri} - data = self.session.get(url, params=params).text + self._oauth1_authorization_flow( + "https://www.flickr.com/services/oauth/request_token", + "https://www.flickr.com/services/oauth/authorize", + "https://www.flickr.com/services/oauth/access_token", + ) - data = urllib.parse.parse_qs(data) - self.session.params["oauth_token"] = token = data["oauth_token"][0] - self.session.token_secret = data["oauth_token_secret"][0] - # Get the User's Authorization - url = "https://www.flickr.com/services/oauth/authorize" - params = {"oauth_token": token, "perms": "read"} - data = self.open(url, params) +class OAuthReddit(OAuthBase): + subcategory = "reddit" + pattern = ["oauth:reddit$"] - # Exchange the Request Token for an Access Token - url = "https://www.flickr.com/services/oauth/access_token" - data = self.session.get(url, params=data).text + def items(self): + yield Message.Version, 1 - data = urllib.parse.parse_qs(data) - self.send(FLICKR_MSG_TEMPLATE.format( - token=data["oauth_token"][0], - token_secret=data["oauth_token_secret"][0])) + self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT + self._oauth2_authorization_code_grant( + reddit.RedditAPI.CLIENT_ID, + "", + "https://www.reddit.com/api/v1/authorize", + "https://www.reddit.com/api/v1/access_token", + "read", + ) + + +class OAuthTumblr(OAuthBase): + subcategory = "tumblr" + pattern = ["oauth:tumblr$"] + + def __init__(self, match): + OAuthBase.__init__(self, match) + self.session = util.OAuthSession( + self.session, + tumblr.TumblrAPI.API_KEY, + tumblr.TumblrAPI.API_SECRET, + ) + + def items(self): + yield Message.Version, 1 + + self._oauth1_authorization_flow( + "https://www.tumblr.com/oauth/request_token", + "https://www.tumblr.com/oauth/authorize", + "https://www.tumblr.com/oauth/access_token", + ) + + +OAUTH1_MSG_TEMPLATE = """ +Your Access Token and Access Token Secret are + +{token} +{token_secret} + +Put these values into your configuration file as +'extractor.{category}.access-token' and +'extractor.{category}.access-token-secret'. + +Example: +{{ + "extractor": {{ + "{category}": {{ + "access-token": "{token}", + "access-token-secret": "{token_secret}" + }} + }} +}} +""" OAUTH2_MSG_TEMPLATE = """ @@ -210,23 +263,3 @@ Example: }} }} """ - -FLICKR_MSG_TEMPLATE = """ -Your Access Token and Access Token Secret are - -{token} -{token_secret} - -Put these values into your configuration file as -'extractor.flickr.access-token' and 'extractor.flickr.access-token-secret'. - -Example: -{{ - "extractor": {{ - "flickr": {{ - "access-token": "{token}", - "access-token-secret": "{token_secret}" - }} - }} -}} -""" diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 9509577c..dfdd71cd 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2017 Mike Fährmann +# Copyright 2016-2018 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,7 +9,7 @@ """Extract images from https://www.tumblr.com/""" from .common import Extractor, Message -from .. import text, exception +from .. import text, util, exception from ..cache import memcache import re @@ -202,9 +202,20 @@ class TumblrTagExtractor(TumblrExtractor): class TumblrAPI(): """Minimal interface for the Tumblr API v2""" API_KEY = "O3hU2tMi5e4Qs5t3vezEi6L0qRORJ5y9oUpSGsrWu8iA3UCc3B" + API_SECRET = "sFdsK3PDdP2QpYMRAoq0oDnw0sFS24XigXmdfnaeNZpJpqAn03" def __init__(self, extractor): - self.api_key = extractor.config("api-key", TumblrAPI.API_KEY) + self.api_key = extractor.config("api-key", self.API_KEY) + api_secret = extractor.config("api-secret", self.API_SECRET) + token = extractor.config("access-token") + token_secret = extractor.config("access-token-secret") + if token and token_secret: + self.session = util.OAuthSession( + extractor.session, + self.api_key, api_secret, token, token_secret) + self.api_key = None + else: + self.session = extractor.session self.params = {"offset": 0, "limit": 50, "reblog_info": "true"} self.extractor = extractor @@ -219,12 +230,12 @@ class TumblrAPI(): return self._pagination(blog, "posts", params) def _call(self, blog, endpoint, params): - params["api_key"] = self.api_key + if self.api_key: + params["api_key"] = self.api_key url = "https://api.tumblr.com/v2/blog/{}.tumblr.com/{}".format( blog, endpoint) - response = self.extractor.request( - url, params=params, fatal=False).json() + response = self.session.get(url, params=params).json() if response["meta"]["status"] == 404: raise exception.NotFoundError("user or post") elif response["meta"]["status"] != 200: diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 0a8fee0d..6d7aea42 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017 Mike Fährmann +# Copyright 2017-2018 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -486,21 +486,26 @@ class OAuthSession(): params.update(self.params) params["oauth_nonce"] = self.nonce(16) params["oauth_timestamp"] = int(time.time()) - params["oauth_signature"] = self.signature(url, params) - return self.session.get(url, params=params) + return self.session.get(url + self.sign(url, params)) - def signature(self, url, params): - """Generate 'oauth_signature' value""" - query = urllib.parse.urlencode(sorted(params.items())) + def sign(self, url, params): + """Generate 'oauth_signature' value and return query string""" + query = urllib.parse.urlencode( + sorted(params.items()), quote_via=self.quote) message = self.concat("GET", url, query).encode() key = self.concat(self.consumer_secret, self.token_secret).encode() signature = hmac.new(key, message, hashlib.sha1).digest() - return base64.b64encode(signature).decode() + return "?{}&oauth_signature={}".format( + query, self.quote(base64.b64encode(signature).decode())) @staticmethod def concat(*args): - return "&".join(urllib.parse.quote(item, "") for item in args) + return "&".join(OAuthSession.quote(item) for item in args) @staticmethod def nonce(N, alphabet=string.ascii_letters): return "".join(random.choice(alphabet) for _ in range(N)) + + @staticmethod + def quote(value, _=None, encoding=None, errors=None): + return urllib.parse.quote(value, "~", encoding, errors) diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py index 7d6cd429..c6b0bc13 100755 --- a/scripts/build_supportedsites.py +++ b/scripts/build_supportedsites.py @@ -88,6 +88,7 @@ AUTH_MAP = { "reddit" : "Optional (OAuth)", "sankaku" : "Optional", "seiga" : "Required", + "tumblr" : "Optional (OAuth)", } IGNORE_LIST = (