diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 3be862b3..86a86d05 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -91,7 +91,7 @@ SmugMug https://www.smugmug.com/ |smugmug-C| The /b/ Archive https://thebarchive.com/ Threads Tsumino https://www.tsumino.com/ Galleries, Search Results Optional Tumblr https://www.tumblr.com/ Images from Users, Likes, Posts, Tag-Searches Optional (OAuth) -Twitter https://twitter.com/ Media Timelines, Timelines, Tweets +Twitter https://twitter.com/ Media Timelines, Timelines, Tweets Optional Wallhaven https://alpha.wallhaven.cc/ individual Images, Search Results Optional Warosu https://warosu.org/ Threads Weibo https://www.weibo.com/ Images from Users, Images from Statuses @@ -99,7 +99,7 @@ WikiArt.org https://www.wikiart.org/ Artists, Artworks World Three http://www.slide.world-three.org/ Chapters, Manga XVideos https://www.xvideos.com/ Images from Users, Galleries Yandere https://yande.re/ Pools, Popular Images, Posts, Tag-Searches -yaplog! https://yaplog.jp/ Images from Users, Posts +yaplog! https://yaplog.jp/ Blogs, Posts |yuki-S| https://yuki.la/ Threads Acidimg https://acidimg.cc/ individual Images Imagetwist https://imagetwist.com/ individual Images diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 201b3a62..0d496aff 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -9,7 +9,8 @@ """Extract images from https://twitter.com/""" from .common import Extractor, Message -from .. import text +from .. import text, exception +from ..cache import cache class TwitterExtractor(Extractor): @@ -27,6 +28,7 @@ class TwitterExtractor(Extractor): self.videos = self.config("videos", False) def items(self): + self.login() yield Message.Version, 1 yield Message.Directory, self.metadata() @@ -54,6 +56,35 @@ class TwitterExtractor(Extractor): def tweets(self): """Yield HTML content of all relevant tweets""" + def login(self): + username, password = self._get_auth_info() + if username: + self._update_cookies(self._login_impl(username, password)) + + @cache(maxage=360*24*3600, keyarg=1) + def _login_impl(self, username, password): + self.log.info("Logging in as %s", username) + + page = self.request(self.root + "/login").text + pos = page.index('name="authenticity_token"') + token = text.extract(page, 'value="', '"', pos-80)[0] + + url = self.root + "/sessions" + data = { + "session[username_or_email]": username, + "session[password]" : password, + "authenticity_token" : token, + "ui_metrics" : '{"rf":{},"s":""}', + "scribe_log" : "", + "redirect_after_login" : "", + "remember_me" : "1", + } + response = self.request(url, method="POST", data=data) + + if "/error" in response.url: + raise exception.AuthenticationError() + return self.session.cookies + @staticmethod def _data_from_tweet(tweet): data = text.extract_all(tweet, ( diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index acfc0d0d..c975be29 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -110,6 +110,7 @@ AUTH_MAP = { "smugmug" : "Optional (OAuth)", "tsumino" : "Optional", "tumblr" : "Optional (OAuth)", + "twitter" : "Optional", "wallhaven" : "Optional", } diff --git a/test/test_results.py b/test/test_results.py index bcf7b824..a7df5021 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -227,6 +227,7 @@ def setup_test_config(): config.set(("extractor", "nijie", "username"), email) config.set(("extractor", "seiga", "username"), email) config.set(("extractor", "danbooru", "username"), None) + config.set(("extractor", "twitter" , "username"), None) config.set(("extractor", "deviantart", "client-id"), "7777") config.set(("extractor", "deviantart", "client-secret"),