From 173984475898902492d9cd63db19a9c9dc283085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Mon, 30 Jul 2018 17:50:22 +0200 Subject: [PATCH 1/5] Fix login by allowing more http headers Because Instagram is bombarding over hundret of headers on login it is necessary to increase http.client._MAXHEADERS. Fixes #152. --- instaloader/instaloadercontext.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/instaloader/instaloadercontext.py b/instaloader/instaloadercontext.py index 395c17e..d7b65a3 100644 --- a/instaloader/instaloadercontext.py +++ b/instaloader/instaloadercontext.py @@ -175,6 +175,9 @@ class InstaloaderContext: :raises InvalidArgumentException: If the provided username does not exist. :raises BadCredentialsException: If the provided password is wrong. :raises ConnectionException: If connection to Instagram failed.""" + import http.client + # pylint:disable=protected-access + http.client._MAXHEADERS = 200 session = requests.Session() session.cookies.update({'sessionid': '', 'mid': '', 'ig_pr': '1', 'ig_vw': '1920', 'csrftoken': '', From 50a5330fecb1272487fdd600416354c83ef8922f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 2 Aug 2018 16:20:27 +0200 Subject: [PATCH 2/5] Change download of stories when not using :stories If using --stories or --stories-only the stories got donwloaded along with the profiles one by one. Now, the stories get downloaded in a similar aproach like when using the :stories target, i.e. download_stories() gets only called once. Profile.has_highlight_reels is broken and now always returns true. This fixes #153. --- instaloader/__main__.py | 49 ++++++++++++++++++++---------------- instaloader/instaloader.py | 51 ++++++++++++++++++++++++-------------- instaloader/structures.py | 7 +++--- 3 files changed, 64 insertions(+), 43 deletions(-) diff --git a/instaloader/__main__.py b/instaloader/__main__.py index 11f1707..8de51a5 100644 --- a/instaloader/__main__.py +++ b/instaloader/__main__.py @@ -121,8 +121,9 @@ def _main(instaloader: Instaloader, targetlist: List[str], if target[0] == '@': instaloader.context.log("Retrieving followees of %s..." % target[1:]) profile = Profile.from_username(instaloader.context, target[1:]) - followees = profile.get_followees() - profiles.update([followee.username for followee in followees]) + for followee in profile.get_followees(): + instaloader.save_profile_id(followee) + profiles.add(followee) elif target[0] == '#': instaloader.download_hashtag(hashtag=target[1:], max_count=max_count, fast_update=fast_update, post_filter=post_filter) @@ -135,26 +136,32 @@ def _main(instaloader: Instaloader, targetlist: List[str], instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count, post_filter=post_filter) else: - profiles.add(target) + profiles.add(instaloader.check_profile_id(target)) if len(profiles) > 1: - instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), ' '.join(profiles))) - # Iterate through profiles list and download them - for target in profiles: - with instaloader.context.error_catcher(target): - try: - instaloader.download_profile(target, profile_pic, profile_pic_only, fast_update, - stories, stories_only, post_filter=post_filter, - storyitem_filter=storyitem_filter) - except ProfileNotExistsException as err: - if instaloader.context.is_logged_in: - instaloader.context.log(err) - instaloader.context.log("Trying again anonymously, helps in case you are just blocked.") - with instaloader.anonymous_copy() as anonymous_loader: - with instaloader.context.error_catcher(): - anonymous_loader.download_profile(target, profile_pic, profile_pic_only, - fast_update, post_filter=post_filter) - else: - raise + instaloader.context.log("Downloading {} profiles: {}".format(len(profiles), + ' '.join([p.username for p in profiles]))) + if not stories_only: + # Iterate through profiles list and download them + for target in profiles: + with instaloader.context.error_catcher(target): + try: + instaloader.download_profile(target, profile_pic, profile_pic_only, + fast_update, post_filter=post_filter) + except ProfileNotExistsException as err: + if instaloader.context.is_logged_in and not stories_only: + instaloader.context.log(err) + instaloader.context.log("Trying again anonymously, helps in case you are just blocked.") + with instaloader.anonymous_copy() as anonymous_loader: + with instaloader.context.error_catcher(): + anonymous_loader.download_profile(target, profile_pic, profile_pic_only, + fast_update, post_filter=post_filter) + else: + raise + if stories or stories_only: + with instaloader.context.error_catcher("Download stories"): + instaloader.context.log("Downloading stories") + instaloader.download_stories(userids=list(profiles), fast_update=fast_update, + filename_target=None, storyitem_filter=storyitem_filter) except KeyboardInterrupt: print("\nInterrupted by user.", file=sys.stderr) # Save session if it is useful diff --git a/instaloader/instaloader.py b/instaloader/instaloader.py index 908fd75..41d819f 100644 --- a/instaloader/instaloader.py +++ b/instaloader/instaloader.py @@ -11,7 +11,7 @@ from contextlib import contextmanager, suppress from datetime import datetime, timezone from functools import wraps from io import BytesIO -from typing import Callable, Iterator, List, Optional, Any +from typing import Any, Callable, Iterator, List, Optional, Union from .exceptions import * from .instaloadercontext import InstaloaderContext @@ -410,25 +410,26 @@ class Instaloader: @_requires_login def download_stories(self, - userids: Optional[List[int]] = None, + userids: Optional[List[Union[int, Profile]]] = None, fast_update: bool = False, - filename_target: str = ':stories', + filename_target: Optional[str] = ':stories', storyitem_filter: Optional[Callable[[StoryItem], bool]] = None) -> None: """ Download available stories from user followees or all stories of users whose ID are given. Does not mark stories as seen. To use this, one needs to be logged in - :param userids: List of user IDs to be processed in terms of downloading their stories + :param userids: List of user IDs or Profiles to be processed in terms of downloading their stories :param fast_update: If true, abort when first already-downloaded picture is encountered :param filename_target: Replacement for {target} in dirname_pattern and filename_pattern + or None if profile name should be used instead :param storyitem_filter: function(storyitem), which returns True if given StoryItem should be downloaded """ if not userids: self.context.log("Retrieving all visible stories...") - for user_story in self.get_stories(userids): + for user_story in self.get_stories([p if isinstance(p, int) else p.userid for p in userids]): name = user_story.owner_username self.context.log("Retrieving stories from profile {}.".format(name)) totalcount = user_story.itemcount @@ -440,7 +441,7 @@ class Instaloader: self.context.log("[%3i/%3i] " % (count, totalcount), end="", flush=True) count += 1 with self.context.error_catcher('Download story from user {}'.format(name)): - downloaded = self.download_storyitem(item, filename_target) + downloaded = self.download_storyitem(item, filename_target if filename_target else name) if fast_update and not downloaded: break @@ -611,6 +612,24 @@ class Instaloader: if fast_update and not downloaded: break + def _get_id_filename(self, profile_name: str) -> str: + if ((format_string_contains_key(self.dirname_pattern, 'profile') or + format_string_contains_key(self.dirname_pattern, 'target'))): + return '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(), + target=profile_name.lower())) + else: + return '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower()) + + def save_profile_id(self, profile: Profile): + """ + Store ID of profile locally. + """ + os.makedirs(self.dirname_pattern.format(profile=profile.username, + target=profile.username), exist_ok=True) + with open(self._get_id_filename(profile.username), 'w') as text_file: + text_file.write(str(profile.userid) + "\n") + self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile.username)) + def check_profile_id(self, profile_name: str) -> Profile: """ Consult locally stored ID of profile with given name, check whether ID matches and whether name @@ -623,12 +642,7 @@ class Instaloader: with suppress(ProfileNotExistsException): profile = Profile.from_username(self.context, profile_name) profile_exists = profile is not None - if ((format_string_contains_key(self.dirname_pattern, 'profile') or - format_string_contains_key(self.dirname_pattern, 'target'))): - id_filename = '{0}/id'.format(self.dirname_pattern.format(profile=profile_name.lower(), - target=profile_name.lower())) - else: - id_filename = '{0}/{1}_id'.format(self.dirname_pattern.format(), profile_name.lower()) + id_filename = self._get_id_filename(profile_name) try: with open(id_filename, 'rb') as id_file: profile_id = int(id_file.read()) @@ -657,15 +671,11 @@ class Instaloader: except (FileNotFoundError, ValueError): pass if profile_exists: - os.makedirs(self.dirname_pattern.format(profile=profile_name.lower(), - target=profile_name.lower()), exist_ok=True) - with open(id_filename, 'w') as text_file: - text_file.write(str(profile.userid) + "\n") - self.context.log("Stored ID {0} for profile {1}.".format(profile.userid, profile_name)) + self.save_profile_id(profile) return profile raise ProfileNotExistsException("Profile {0} does not exist.".format(profile_name)) - def download_profile(self, profile_name: str, + def download_profile(self, profile_name: Union[str, Profile], profile_pic: bool = True, profile_pic_only: bool = False, fast_update: bool = False, download_stories: bool = False, download_stories_only: bool = False, @@ -676,7 +686,10 @@ class Instaloader: # Get profile main page json # check if profile does exist or name has changed since last download # and update name and json data if necessary - profile = self.check_profile_id(profile_name.lower()) + if isinstance(profile_name, str): + profile = self.check_profile_id(profile_name.lower()) + else: + profile = profile_name profile_name = profile.username diff --git a/instaloader/structures.py b/instaloader/structures.py index 7e19c9d..d336f55 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -526,10 +526,11 @@ class Profile: @property def has_highlight_reels(self) -> bool: """ - This becomes `True` if the :class:`Profile` has any stories currently available, - even if not viewable by the viewer. + Always returns `True` since :issue:`153`. + + Before broken, this indicated whether the :class:`Profile` had available stories. """ - return self._iphone_struct['has_highlight_reels'] + return True @property def has_public_story(self) -> bool: From bc29ffbd44e87c975e089e87e8baf6f4aadac808 Mon Sep 17 00:00:00 2001 From: Lars Lindqvist Date: Thu, 26 Jul 2018 19:51:33 +0200 Subject: [PATCH 3/5] Add meth:get_tagged_posts to Profile --- instaloader/structures.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/instaloader/structures.py b/instaloader/structures.py index d336f55..8d9312e 100644 --- a/instaloader/structures.py +++ b/instaloader/structures.py @@ -610,6 +610,16 @@ class Profile: self._rhx_gis, self._metadata('edge_saved_media'))) + def get_tagged_posts(self) -> Iterator[Post]: + """Retrieve all posts where a profile is tagged.""" + self._obtain_metadata() + yield from (Post(self._context, node, self) for node in + self._context.graphql_node_list("e31a871f7301132ceaab56507a66bbb7", + {'id': self.userid}, + 'https://www.instagram.com/{0}/'.format(self.username), + lambda d: d['data']['user']['edge_user_to_photos_of_you'], + self._rhx_gis)) + def get_followers(self) -> Iterator['Profile']: """ Retrieve list of followers of given profile. From 1b5c12c8faf360eb5941b071ca47061cf2e4bf6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 2 Aug 2018 16:48:06 +0200 Subject: [PATCH 4/5] Add unittest for Profile.get_tagged_posts() --- test/instaloader_unittests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/instaloader_unittests.py b/test/instaloader_unittests.py index 6d828a4..5f2a0fb 100644 --- a/test/instaloader_unittests.py +++ b/test/instaloader_unittests.py @@ -76,6 +76,11 @@ class TestInstaloaderAnonymously(unittest.TestCase): self.assertEqual(post, post2) break + def test_public_profile_tagged_paging(self): + for post in islice(instaloader.Profile.from_username(self.L.context, PUBLIC_PROFILE).get_tagged_posts(), + PAGING_MAX_COUNT): + print(post) + class TestInstaloaderLoggedIn(TestInstaloaderAnonymously): From 92653dcdb9a6d833ff4a56affce313596d5737a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Koch-Kramer?= Date: Thu, 2 Aug 2018 16:55:59 +0200 Subject: [PATCH 5/5] Release of version 4.0.6 --- instaloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instaloader/__init__.py b/instaloader/__init__.py index af0682e..96f1b19 100644 --- a/instaloader/__init__.py +++ b/instaloader/__init__.py @@ -1,7 +1,7 @@ """Download pictures (or videos) along with their captions and other metadata from Instagram.""" -__version__ = '4.0.5' +__version__ = '4.0.6' try: