From c59b98c81bc48fa850f9f0ea5bd70db7ef1f6906 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Mon, 9 Dec 2019 20:21:28 +0100
Subject: [PATCH] [downloader:http] improve rate limit handling

- Move the download "logic" with rate limit checks into its own
  method that only gets used if a rate limit should be enforced
- Fix an issue where suspending gallery-dl during a download would
  basically ignore the rate limit for the remaining download when
  resuming its execution.
---
 gallery_dl/downloader/http.py | 36 +++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 2f43430b..e66cb60f 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -40,11 +40,13 @@ class HttpDownloader(DownloaderBase):
             self.retries = float("inf")
         if self.rate:
             rate = text.parse_bytes(self.rate)
-            if not rate:
+            if rate:
+                if rate < self.chunk_size:
+                    self.chunk_size = rate
+                self.rate = rate
+                self.receive = self._receive_rate
+            else:
                 self.log.warning("Invalid rate limit (%r)", self.rate)
-            elif rate < self.chunk_size:
-                self.chunk_size = rate
-            self.rate = rate
 
     def download(self, url, pathfmt):
         try:
@@ -170,20 +172,26 @@ class HttpDownloader(DownloaderBase):
         return True
 
     def receive(self, response, file):
-        if self.rate:
-            total = 0            # total amount of bytes received
-            start = time.time()  # start time
+        for data in response.iter_content(self.chunk_size):
+            file.write(data)
+
+    def _receive_rate(self, response, file):
+        t1 = time.time()
+        rt = self.rate
 
         for data in response.iter_content(self.chunk_size):
             file.write(data)
 
-            if self.rate:
-                total += len(data)
-                expected = total / self.rate  # expected elapsed time
-                delta = time.time() - start   # actual elapsed time since start
-                if delta < expected:
-                    # sleep if less time passed than expected
-                    time.sleep(expected - delta)
+            t2 = time.time()           # current time
+            actual = t2 - t1           # actual elapsed time
+            expected = len(data) / rt  # expected elapsed time
+
+            if actual < expected:
+                # sleep if less time elapsed than expected
+                time.sleep(expected - actual)
+                t1 = time.time()
+            else:
+                t1 = t2
 
     def get_extension(self, response):
         mtype = response.headers.get("Content-Type", "image/jpeg")