Merge branch 'master' into master

instaloader/__init__.py

@@ -1,7 +1,7 @@
 """Download pictures (or videos) along with their captions and other metadata from Instagram."""
 
 
-__version__ = '4.0.6'
+__version__ = '4.0.7'
 
 
 try:

instaloader/__main__.py

@@ -91,6 +91,7 @@ def _main(instaloader: Instaloader, targetlist: List[str],
         instaloader.context.log("Logged in as %s." % username)
     # Try block for KeyboardInterrupt (save session on ^C)
     profiles = set()
+    anonymous_retry_profiles = set()
     try:
         # Generate set of profiles, already downloading non-profile targets
         for target in targetlist:
@@ -137,7 +138,28 @@ def _main(instaloader: Instaloader, targetlist: List[str],
                     instaloader.download_saved_posts(fast_update=fast_update, max_count=max_count,
                                                      post_filter=post_filter)
                 else:
-                    profiles.add(instaloader.check_profile_id(target))
+                    try:
+                        profile = instaloader.check_profile_id(target)
+                        if instaloader.context.is_logged_in and profile.has_blocked_viewer:
+                            if not stories_only and not profile.is_private:
+                                raise ProfileNotExistsException("{} blocked you; But she's public, "
+                                                                "so we just download her anonymously.".format(target))
+                            else:
+                                instaloader.context.error("{} blocked you.".format(target))
+                        else:
+                            profiles.add(profile)
+                    except ProfileNotExistsException as err:
+                        # Not only our profile.has_blocked_viewer condition raises ProfileNotExistsException,
+                        # check_profile_id() also does, since access to blocked profile may be responded with 404.
+                        if instaloader.context.is_logged_in and not stories_only:
+                            instaloader.context.log(err)
+                            instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
+                            with instaloader.anonymous_copy() as anonymous_loader:
+                                with instaloader.context.error_catcher():
+                                    anonymous_retry_profiles.add(anonymous_loader.check_profile_id(target))
+                                    instaloader.context.log("Looks good.")
+                        else:
+                            raise
         if len(profiles) > 1:
             instaloader.context.log("Downloading {} profiles: {}".format(len(profiles),
                                                                          ' '.join([p.username for p in profiles])))
@@ -145,22 +167,20 @@ def _main(instaloader: Instaloader, targetlist: List[str],
             # Iterate through profiles list and download them
             for target in profiles:
                 with instaloader.context.error_catcher(target):
-                    try:
+                    instaloader.download_profile(target, profile_pic, profile_pic_only,
-                        instaloader.download_profile(target, profile_pic, profile_pic_only,
+                                                 fast_update, download_tagged=tagged,
-                                                     fast_update, download_tagged=tagged,
+                                                 download_tagged_only=tagged_only,
-                                                     download_tagged_only=tagged_only, post_filter=post_filter)
+                                                 post_filter=post_filter)
-                    except ProfileNotExistsException as err:
+            if anonymous_retry_profiles:
-                        if instaloader.context.is_logged_in and not stories_only:
+                instaloader.context.log("Downloading anonymously: {}"
-                            instaloader.context.log(err)
+                                        .format(' '.join([p.username for p in anonymous_retry_profiles])))
-                            instaloader.context.log("Trying again anonymously, helps in case you are just blocked.")
+                with instaloader.anonymous_copy() as anonymous_loader:
-                            with instaloader.anonymous_copy() as anonymous_loader:
+                    for target in anonymous_retry_profiles:
-                                with instaloader.context.error_catcher():
+                        with instaloader.context.error_catcher(target):
-                                    anonymous_loader.download_profile(target, profile_pic, profile_pic_only,
+                            anonymous_loader.download_profile(target, profile_pic, profile_pic_only,
-                                                                      fast_update, download_tagged=tagged,
+                                                              fast_update, download_tagged=tagged,
-                                                                      download_tagged_only=tagged_only,
+                                                              download_tagged_only=tagged_only,
-                                                                      post_filter=post_filter)
+                                                              post_filter=post_filter)
-                        else:
-                            raise
         if stories or stories_only:
             with instaloader.context.error_catcher("Download stories"):
                 instaloader.context.log("Downloading stories")

instaloader/instaloader.py

@@ -49,11 +49,11 @@ class _ArbitraryItemFormatter(string.Formatter):
     def __init__(self, item: Any):
         self._item = item
 
-    def get_field(self, field_name, args, kwargs):
+    def get_value(self, key, args, kwargs):
         """Override to substitute {ATTRIBUTE} by attributes of our _item."""
-        if hasattr(self._item, field_name):
+        if hasattr(self._item, key):
-            return self._item.__getattribute__(field_name), None
+            return getattr(self._item, key)
-        return super().get_field(field_name, args, kwargs)
+        return super().get_value(key, args, kwargs)
 
     def format_field(self, value, format_spec):
         """Override :meth:`string.Formatter.format_field` to have our
@@ -396,17 +396,22 @@ class Instaloader:
         :param userids: List of user IDs to be processed in terms of downloading their stories, or None.
         """
 
-        if userids is None:
+        if not userids:
             data = self.context.graphql_query("d15efd8c0c5b23f0ef71f18bf363c704",
                                               {"only_stories": True})["data"]["user"]
             if data is None:
                 raise BadResponseException('Bad stories reel JSON.')
             userids = list(edge["node"]["id"] for edge in data["feed_reels_tray"]["edge_reels_tray_to_reel"]["edges"])
 
-        stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056",
+        def _userid_chunks():
-                                             {"reel_ids": userids, "precomposed_overlay": False})["data"]
+            userids_per_query = 100
+            for i in range(0, len(userids), userids_per_query):
+                yield userids[i:i + userids_per_query]
 
-        yield from (Story(self.context, media) for media in stories['reels_media'])
+        for userid_chunk in _userid_chunks():
+            stories = self.context.graphql_query("bf41e22b1c4ba4c9f31b844ebb7d9056",
+                                                 {"reel_ids": userid_chunk, "precomposed_overlay": False})["data"]
+            yield from (Story(self.context, media) for media in stories['reels_media'])
 
     @_requires_login
     def download_stories(self,
@@ -428,8 +433,10 @@ class Instaloader:
 
         if not userids:
             self.context.log("Retrieving all visible stories...")
+        else:
+            userids = [p if isinstance(p, int) else p.userid for p in userids]
 
-        for user_story in self.get_stories([p if isinstance(p, int) else p.userid for p in userids]):
+        for user_story in self.get_stories(userids):
             name = user_story.owner_username
             self.context.log("Retrieving stories from profile {}.".format(name))
             totalcount = user_story.itemcount

instaloader/instaloadercontext.py

@@ -9,6 +9,7 @@ import textwrap
 import time
 import urllib.parse
 from contextlib import contextmanager
+from datetime import datetime, timedelta
 from typing import Any, Callable, Dict, Iterator, Optional
 
 import requests
@@ -246,7 +247,8 @@ class InstaloaderContext:
         if is_graphql_query and not query_not_limited:
             waittime = graphql_query_waittime()
             if waittime > 0:
-                self.log('\nToo many queries in the last time. Need to wait {} seconds.'.format(waittime))
+                self.log('\nToo many queries in the last time. Need to wait {} seconds, until {:%H:%M}.'
+                         .format(waittime, datetime.now() + timedelta(seconds=waittime)))
                 time.sleep(waittime)
             if self.query_timestamps is not None:
                 self.query_timestamps.append(time.monotonic())
@@ -301,7 +303,8 @@ class InstaloaderContext:
                     if is_graphql_query:
                         waittime = graphql_query_waittime(untracked_queries=True)
                         if waittime > 0:
-                            self.log('The request will be retried in {} seconds.'.format(waittime))
+                            self.log('The request will be retried in {} seconds, at {:%H:%M}.'
+                                     .format(waittime, datetime.now() + timedelta(seconds=waittime)))
                             time.sleep(waittime)
                 self._sleep()
                 return self.get_json(path=path, params=params, host=host, session=sess, _attempt=_attempt + 1)

instaloader/structures.py

@@ -526,7 +526,8 @@ class Profile:
     @property
     def has_highlight_reels(self) -> bool:
         """
-        Always returns `True` since :issue:`153`.
+        .. deprecated:: 4.0.6
+           Always returns `True` since :issue:`153`.
 
         Before broken, this indicated whether the :class:`Profile` had available stories.
         """