add options to control usage of .part files (#29)

- '--no-part' command line option to disable them
- 'downloader.http.part' and 'downloader.text.part' config options

Disabling .part files restores the behaviour of the old downloader
implementation.

docs/configuration.rst

@@ -92,6 +92,21 @@ Description Controls the progress indicator when *gallery-dl* is run with
 Downloader Options
 ==================
 
+downloader.part
+---------------
+=========== =====
+Type        ``bool``
+Default     ``true``
+Description Controls the use of ``.part`` files during file downloads.
+
+            * ``true``: Write downloaded data into ``.part`` files and rename
+              them upon download completion. This mode additionally supports
+              resuming incomplete downloads.
+            * ``false``: Do not use ``.part`` files and write data directly
+              into the actual output files.
+=========== =====
+
+
 downloader.http.retries
 -----------------------
 =========== =====

docs/gallery-dl.conf

@@ -3,6 +3,7 @@
     "netrc": false,
     "downloader":
     {
+        "part": true,
         "http":
         {
             "retries": 5,

gallery_dl/downloader/common.py

@@ -9,31 +9,41 @@
 """Common classes and constants used by downloader modules."""
 
 import os
-import os.path
-import logging
 import time
+import logging
 
 
 class DownloaderBase():
     """Base class for downloaders"""
     retries = 1
     mode = "b"
+    part = True
 
     def __init__(self, session, output):
         self.session = session
         self.out = output
         self.log = logging.getLogger("download")
+        self.downloading = False
 
     def download(self, url, pathfmt):
         """Download the resource at 'url' and write it to a file-like object"""
+        try:
+            self.download_impl(url, pathfmt)
+        finally:
+            # remove file from incomplete downloads
+            if self.downloading and not self.part:
+                try:
+                    os.remove(pathfmt.realpath)
+                except (OSError, AttributeError):
+                    pass
+
+    def download_impl(self, url, pathfmt):
+        """Actual implementaion of the download process"""
         tries = 0
         msg = ""
 
-        if not pathfmt.has_extension:
-            pathfmt.set_extension("part", False)
-            partpath = pathfmt.realpath
-        else:
-            partpath = pathfmt.realpath + ".part"
+        if self.part:
+            pathfmt.part_enable()
 
         while True:
             if tries:
@@ -45,12 +55,7 @@ class DownloaderBase():
             self.reset()
 
             # check for .part file
-            filesize = 0
-            if os.path.isfile(partpath):
-                try:
-                    filesize = os.path.getsize(partpath)
-                except OSError:
-                    pass
+            filesize = pathfmt.part_size()
 
             # connect to (remote) source
             try:
@@ -78,7 +83,8 @@ class DownloaderBase():
                     return True
 
             self.out.start(pathfmt.path)
-            with open(partpath, mode) as file:
+            self.downloading = True
+            with pathfmt.open(mode) as file:
                 # download content
                 try:
                     self.receive(file)
@@ -95,7 +101,9 @@ class DownloaderBase():
                     continue
             break
 
-        os.rename(partpath, pathfmt.realpath)
+        self.downloading = False
+        if self.part:
+            pathfmt.part_move()
         self.out.success(pathfmt.path, tries)
         return True
 

gallery_dl/downloader/http.py

@@ -13,10 +13,15 @@ from .common import DownloaderBase
 from .. import config, util
 
 
+def _conf(key, default=None):
+    return config.interpolate(("downloader", "http", key), default)
+
+
 class Downloader(DownloaderBase):
-    retries = config.interpolate(("downloader", "http", "retries",), 5)
-    timeout = config.interpolate(("downloader", "http", "timeout",), 30)
-    verify = config.interpolate(("downloader", "http", "verify",), True)
+    retries = _conf("retries", 5)
+    timeout = _conf("timeout", 30)
+    verify = _conf("verify", True)
+    part = _conf("part", True)
 
     def __init__(self, session, output):
         DownloaderBase.__init__(self, session, output)

gallery_dl/downloader/text.py

@@ -9,9 +9,11 @@
 """Downloader module for text: URLs"""
 
 from .common import DownloaderBase
+from .. import config
 
 
 class Downloader(DownloaderBase):
+    part = config.interpolate(("downloader", "text", "part"), True)
     mode = "t"
 
     def __init__(self, session, output):

gallery_dl/option.py

@@ -145,6 +145,11 @@ def build_parser():
         metavar="SECONDS", action=ConfigAction, dest="timeout", type=float,
         help="Timeout for HTTP connections (defaut: 30s)",
     )
+    downloader.add_argument(
+        "--no-part",
+        action=ConfigConstAction, nargs=0, dest="part", const=False,
+        help="Do not use .part files",
+    )
     downloader.add_argument(
         "--abort-on-skip",
         action=ConfigConstAction, nargs=0, dest="skip", const="abort",

gallery_dl/util.py

@@ -15,6 +15,7 @@ import hmac
 import time
 import base64
 import random
+import shutil
 import string
 import _string
 import hashlib
@@ -319,7 +320,7 @@ class PathFormat():
         self.has_extension = False
         self.keywords = {}
         self.directory = self.realdirectory = ""
-        self.path = self.realpath = ""
+        self.path = self.realpath = self.partpath = ""
 
         bdir = extractor.config("base-directory", (".", "gallery-dl"))
         if not isinstance(bdir, str):
@@ -335,8 +336,8 @@ class PathFormat():
             self.exists = lambda: False
 
     def open(self, mode="wb"):
-        """Open file to 'realpath' and return a corresponding file object"""
-        return open(self.realpath, mode)
+        """Open file and return a corresponding file object"""
+        return open(self.partpath or self.realpath, mode)
 
     def exists(self):
         """Return True if 'path' is complete and refers to an existing path"""
@@ -387,6 +388,29 @@ class PathFormat():
         self.path = self.directory + filename
         self.realpath = self.realdirectory + filename
 
+    def part_enable(self):
+        if self.has_extension:
+            self.partpath = self.realpath + ".part"
+        else:
+            self.set_extension("part", False)
+            self.partpath = self.realpath
+
+    def part_size(self):
+        if self.partpath and os.path.isfile(self.partpath):
+            try:
+                return os.path.getsize(self.partpath)
+            except OSError:
+                pass
+        return 0
+
+    def part_move(self):
+        try:
+            os.rename(self.partpath, self.realpath)
+            return
+        except OSError:
+            pass
+        shutil.copyfile(self.partpath, self.realpath)
+
     def _exists_abort(self):
         if self.has_extension and os.path.exists(self.realpath):
             raise exception.StopExtraction()