gallery-dl/gallery_dl/downloader/http.py

# -*- coding: utf-8 -*-

# Copyright 2014, 2015 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

"""Downloader module for http urls"""

from .common import BasicDownloader
import time
import requests

class Downloader(BasicDownloader):

    def __init__(self):
        BasicDownloader.__init__(self)
        self.session = requests.session()

    def download_impl(self, url, file):
        tries = 0
        while True:
            # try to connect to remote source
            try:
                response = self.session.get(url, stream=True, verify=True)
            except requests.exceptions.ConnectionError as exptn:
                tries += 1
                self.print_error(file, exptn, tries, self.max_tries)
                time.sleep(1)
                if tries == self.max_tries:
                    raise
                continue

            # reject error-status-codes
            if response.status_code != requests.codes.ok:
                tries += 1
                self.print_error(file, 'HTTP status "{} {}"'.format(
                    response.status_code, response.reason), tries, self.max_tries)
                if response.status_code == 404:
                    return self.max_tries
                time.sleep(1)
                if tries == 5:
                    response.raise_for_status()
                continue

            # everything ok -- proceed to download
            break

        for data in response.iter_content(16384):
            file.write(data)
        return tries

    def set_headers(self, headers):
        """Set headers for http requests"""
        self.set_dict(self.session.headers, headers)

    def set_cookies(self, cookies):
        """Set cookies for http requests"""
        self.set_dict(self.session.cookies, cookies)

    @staticmethod
    def set_dict(dest, src):
        """Copy the contents of dictionary 'src' to 'dest'"""
        dest.clear()
        dest.update(src)