implement a download progress indicator (#1519)

docs/configuration.rst

@@ -2344,6 +2344,19 @@ Description
     alongside the actual output files.
 
 
+downloader.*.progress
+---------------------
+Type
+    ``float``
+Default
+    ``3.0``
+Description
+    Number of seconds until a download progress indicator
+    for the current download is displayed.
+
+    Set this option to ``null`` to disable this indicator.
+
+
 downloader.*.rate
 -----------------
 Type

docs/gallery-dl.conf

@@ -319,6 +319,7 @@
         "mtime": true,
         "part": true,
         "part-directory": null,
+        "progress": 3.0,
         "rate": null,
         "retries": 4,
         "timeout": 30.0,

gallery_dl/downloader/http.py

@@ -31,6 +31,7 @@ class HttpDownloader(DownloaderBase):
         self.downloading = False
 
         self.adjust_extension = self.config("adjust-extensions", True)
+        self.progress = self.config("progress", 3.0)
         self.headers = self.config("headers")
         self.minsize = self.config("filesize-min")
         self.maxsize = self.config("filesize-max")
@@ -63,6 +64,8 @@ class HttpDownloader(DownloaderBase):
                 self.receive = self._receive_rate
             else:
                 self.log.warning("Invalid rate limit (%r)", self.rate)
+        if self.progress is not None:
+            self.receive = self._receive_rate
 
     def download(self, url, pathfmt):
         try:
@@ -202,6 +205,7 @@ class HttpDownloader(DownloaderBase):
             with pathfmt.open(mode) as fp:
                 if file_header:
                     fp.write(file_header)
+                    offset += len(file_header)
                 elif offset:
                     if adjust_extension and \
                             pathfmt.extension in FILE_SIGNATURES:
@@ -210,7 +214,7 @@ class HttpDownloader(DownloaderBase):
 
                 self.out.start(pathfmt.path)
                 try:
-                    self.receive(fp, content)
+                    self.receive(fp, content, size, offset)
                 except (RequestException, SSLError, OpenSSLError) as exc:
                     msg = str(exc)
                     print()
@@ -234,28 +238,42 @@ class HttpDownloader(DownloaderBase):
         return True
 
     @staticmethod
-    def receive(fp, content):
+    def receive(fp, content, bytes_total, bytes_downloaded):
         write = fp.write
         for data in content:
             write(data)
 
-    def _receive_rate(self, fp, content):
-        rt = self.rate
-        t1 = time.time()
+    def _receive_rate(self, fp, content, bytes_total, bytes_downloaded):
+        rate = self.rate
+        progress = self.progress
+        bytes_start = bytes_downloaded
+        write = fp.write
+        t1 = tstart = time.time()
 
         for data in content:
-            fp.write(data)
+            write(data)
 
             t2 = time.time()           # current time
-            actual = t2 - t1           # actual elapsed time
-            expected = len(data) / rt  # expected elapsed time
+            elapsed = t2 - t1          # elapsed time
+            num_bytes = len(data)
 
-            if actual < expected:
-                # sleep if less time elapsed than expected
-                time.sleep(expected - actual)
-                t1 = time.time()
-            else:
-                t1 = t2
+            if progress is not None:
+                bytes_downloaded += num_bytes
+                tdiff = t2 - tstart
+                if tdiff >= progress:
+                    self.out.progress(
+                        bytes_total, bytes_downloaded,
+                        int((bytes_downloaded - bytes_start) / tdiff),
+                    )
+
+            if rate:
+                expected = num_bytes / rate  # expected elapsed time
+                if elapsed < expected:
+                    # sleep if less time elapsed than expected
+                    time.sleep(expected - elapsed)
+                    t2 = time.time()
+
+            t1 = t2
 
     def _find_extension(self, response):
         """Get filename extension from MIME type"""

gallery_dl/downloader/ytdl.py

@@ -42,6 +42,10 @@ class YoutubeDLDownloader(DownloaderBase):
         if raw_options:
             options.update(raw_options)
 
+        self.progress = self.config("progress", 3.0)
+        if self.progress is not None:
+            options["progress_hooks"] = (self._progress_hook,)
+
         if self.config("logging", True):
             options["logger"] = self.log
         self.forward_cookies = self.config("forward-cookies", False)
@@ -56,7 +60,10 @@ class YoutubeDLDownloader(DownloaderBase):
         kwdict = pathfmt.kwdict
 
         ytdl = kwdict.pop("_ytdl_instance", None)
-        if not ytdl:
+        if ytdl:
+            if self.progress is not None and not ytdl._progress_hooks:
+                ytdl.add_progress_hook(self._progress_hook)
+        else:
             ytdl = self.ytdl
             if self.forward_cookies:
                 set_cookie = ytdl.cookiejar.set_cookie
@@ -126,6 +133,15 @@ class YoutubeDLDownloader(DownloaderBase):
             ytdl.process_info(entry)
         return True
 
+    def _progress_hook(self, info):
+        if info["status"] == "downloading" and \
+                info["elapsed"] >= self.progress:
+            self.out.progress(
+                info["total_bytes"],
+                info["downloaded_bytes"],
+                int(info["speed"]),
+            )
+
     @staticmethod
     def _set_outtmpl(ytdl, outtmpl):
         try:

gallery_dl/output.py

@@ -258,6 +258,9 @@ class NullOutput():
     def success(self, path, tries):
         """Print a message indicating the completion of a download"""
 
+    def progress(self, bytes_total, bytes_downloaded, bytes_per_second):
+        """Display download progress"""
+
 
 class PipeOutput(NullOutput):
 
@@ -289,6 +292,19 @@ class TerminalOutput(NullOutput):
     def success(self, path, tries):
         print("\r", self.shorten(CHAR_SUCCESS + path), sep="")
 
+    def progress(self, bytes_total, bytes_downloaded, bytes_per_second):
+        if bytes_total is None:
+            print("\r {:>8} {:>10} \r".format(
+                util.format_value(bytes_downloaded, "B"),
+                util.format_value(bytes_per_second, "B/s"),
+            ), end="")
+        else:
+            print("\r{:>3}% {:>8} {:>10} \r".format(
+                bytes_downloaded * 100 // bytes_total,
+                util.format_value(bytes_downloaded, "B"),
+                util.format_value(bytes_per_second, "B/s"),
+            ), end="")
+
 
 class ColorOutput(TerminalOutput):