From 7aa9fa796a754d5ee44a1a2dd4c2583b57638a04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Tue, 25 Jul 2017 14:59:41 +0200
Subject: [PATCH] code cleanup and fixes

---
 gallery_dl/extractor/batoto.py    |  2 +-
 gallery_dl/extractor/chan.py      |  1 -
 gallery_dl/extractor/common.py    | 44 ++++++++++++++++---------------
 gallery_dl/extractor/exhentai.py  |  2 +-
 gallery_dl/extractor/mangahere.py |  2 +-
 gallery_dl/extractor/nijie.py     |  7 +++--
 gallery_dl/extractor/pixiv.py     |  2 +-
 gallery_dl/extractor/seiga.py     |  7 +++--
 test/test_cookies.py              |  7 +++++
 test/test_extractors.py           |  3 +++
 10 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py
index e91a5aeb..d90de328 100644
--- a/gallery_dl/extractor/batoto.py
+++ b/gallery_dl/extractor/batoto.py
@@ -26,7 +26,7 @@ class BatotoExtractor():
         """Login and set necessary cookies"""
         if self._check_cookies(self.cookienames):
             return
-        username, password = self.auth_info()
+        username, password = self._get_auth_info()
         if username:
             cookies = self._login_impl(username, password)
             for key, value in cookies.items():
diff --git a/gallery_dl/extractor/chan.py b/gallery_dl/extractor/chan.py
index f359ec6b..8a9c5745 100644
--- a/gallery_dl/extractor/chan.py
+++ b/gallery_dl/extractor/chan.py
@@ -79,7 +79,6 @@ class FoolfuukaThreadExtractor(Extractor):
     def items(self):
         op = True
         yield Message.Version, 1
-        yield Message.Headers, self.session.headers
         for post in self.posts():
             if op:
                 yield Message.Directory, post
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index b43a2edd..31e32c05 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -32,20 +32,7 @@ class Extractor():
     def __init__(self):
         self.session = requests.Session()
         self.log = logging.getLogger(self.category)
-
-        cookies = self.config("cookies")
-        if cookies:
-            if isinstance(cookies, dict):
-                setcookie = self.session.cookies.set
-                for name, value in cookies.items():
-                    setcookie(name, value, domain=self.cookiedomain)
-            else:
-                try:
-                    cj = http.cookiejar.MozillaCookieJar()
-                    cj.load(cookies)
-                    self.session.cookies.update(cj)
-                except OSError as exc:
-                    self.log.warning("cookies: %s", exc)
+        self._set_cookies(self.config("cookies"))
 
     def __iter__(self):
         return self.items()
@@ -60,7 +47,13 @@ class Extractor():
         return config.interpolate(
             ("extractor", self.category, self.subcategory, key), default)
 
-    def auth_info(self):
+    def request(self, url, encoding=None, *args, **kwargs):
+        response = safe_request(self.session, url, *args, **kwargs)
+        if encoding:
+            response.encoding = encoding
+        return response
+
+    def _get_auth_info(self):
         """Return authentication information as (username, password) tuple"""
         username = self.config("username")
         password = None
@@ -78,14 +71,23 @@ class Extractor():
 
         return username, password
 
-    def request(self, url, encoding=None, *args, **kwargs):
-        response = safe_request(self.session, url, *args, **kwargs)
-        if encoding:
-            response.encoding = encoding
-        return response
+    def _set_cookies(self, cookies):
+        """Populate the cookiejar with 'cookies'"""
+        if cookies:
+            if isinstance(cookies, dict):
+                setcookie = self.session.cookies.set
+                for name, value in cookies.items():
+                    setcookie(name, value, domain=self.cookiedomain)
+            else:
+                try:
+                    cj = http.cookiejar.MozillaCookieJar()
+                    cj.load(cookies)
+                    self.session.cookies.update(cj)
+                except OSError as exc:
+                    self.log.warning("cookies: %s", exc)
 
     def _check_cookies(self, cookienames, domain=None):
-        """Return True if all 'cookienames' exist in the current session"""
+        """Check if all 'cookienames' are in the session's cookiejar"""
         if not domain and self.cookiedomain:
             domain = self.cookiedomain
         for name in cookienames:
diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py
index 2acfa6c5..1c0257d4 100644
--- a/gallery_dl/extractor/exhentai.py
+++ b/gallery_dl/extractor/exhentai.py
@@ -180,7 +180,7 @@ class ExhentaiGalleryExtractor(Extractor):
         """Login and set necessary cookies"""
         if self._check_cookies(self.cookienames):
             return
-        username, password = self.auth_info()
+        username, password = self._get_auth_info()
         if not username:
             self.log.info("no username given; using e-hentai.org")
             self.root = "https://e-hentai.org"
diff --git a/gallery_dl/extractor/mangahere.py b/gallery_dl/extractor/mangahere.py
index 1d59211a..f6078baa 100644
--- a/gallery_dl/extractor/mangahere.py
+++ b/gallery_dl/extractor/mangahere.py
@@ -63,7 +63,7 @@ class MangahereChapterExtractor(AsynchronousExtractor):
     def get_job_metadata(self, page):
         """Collect metadata for extractor-job"""
         manga, pos = text.extract(page, '<title>', '</title>')
-        chid , pos = text.extract(page, 'h.mhcdn.net/store/manga/', '/', pos)
+        chid , pos = text.extract(page, '.mhcdn.net/store/manga/', '/', pos)
         _    , pos = text.extract(page, '<select class="wid60"', '', pos)
         _    , pos = text.extract(page, '</select>', '', pos)
         count, pos = text.extract(page, '>', '<', pos-30)
diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py
index b335e7fe..007a6ac5 100644
--- a/gallery_dl/extractor/nijie.py
+++ b/gallery_dl/extractor/nijie.py
@@ -63,10 +63,9 @@ class NijieExtractor(AsynchronousExtractor):
 
     def login(self):
         """Login and obtain session cookie"""
-        if self._check_cookies(("nemail", "nlogin")):
-            return
-        username, password = self.auth_info()
-        self.session.cookies = self._login_impl(username, password)
+        if not self._check_cookies(("nemail", "nlogin")):
+            username, password = self._get_auth_info()
+            self.session.cookies = self._login_impl(username, password)
 
     @cache(maxage=30*24*60*60, keyarg=1)
     def _login_impl(self, username, password):
diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py
index 5305f281..bbd352a0 100644
--- a/gallery_dl/extractor/pixiv.py
+++ b/gallery_dl/extractor/pixiv.py
@@ -269,7 +269,7 @@ class PixivAPI():
     def __init__(self, extractor):
         self.session = extractor.session
         self.log = extractor.log
-        self.username, self.password = extractor.auth_info()
+        self.username, self.password = extractor._get_auth_info()
         self.user_info = None
         self.session.headers.update({
             "Referer": "https://www.pixiv.net/",
diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py
index b005f16c..8e0ba1cb 100644
--- a/gallery_dl/extractor/seiga.py
+++ b/gallery_dl/extractor/seiga.py
@@ -48,10 +48,9 @@ class SeigaExtractor(Extractor):
 
     def login(self):
         """Login and set necessary cookies"""
-        if self._check_cookies(("user_session",)):
-            return
-        username, password = self.auth_info()
-        self.session.cookies = self._login_impl(username, password)
+        if not self._check_cookies(("user_session",)):
+            username, password = self._get_auth_info()
+            self.session.cookies = self._login_impl(username, password)
 
     @cache(maxage=7*24*60*60, keyarg=1)
     def _login_impl(self, username, password):
diff --git a/test/test_cookies.py b/test/test_cookies.py
index c85c09c8..faa1817d 100644
--- a/test/test_cookies.py
+++ b/test/test_cookies.py
@@ -43,6 +43,7 @@ class TestCookiejar(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         cls.path.cleanup()
+        config.clear()
 
     def test_cookiefile(self):
         config.set(CKEY, self.cookiefile)
@@ -79,6 +80,9 @@ class TestCookiedict(unittest.TestCase):
         self.cdict = {"NAME1": "VALUE1", "NAME2": "VALUE2"}
         config.set(CKEY, self.cdict)
 
+    def tearDown(self):
+        config.clear()
+
     def test_dict(self):
         cookies = extractor.find("test:").session.cookies
         self.assertEqual(len(cookies), len(self.cdict))
@@ -97,6 +101,9 @@ class TestCookiedict(unittest.TestCase):
 
 class TestCookieLogin(unittest.TestCase):
 
+    def tearDown(self):
+        config.clear()
+
     def test_cookie_login(self):
         extr_cookies = {
             "batoto": ("member_id", "pass_hash"),
diff --git a/test/test_extractors.py b/test/test_extractors.py
index c102b038..47bfe7ad 100644
--- a/test/test_extractors.py
+++ b/test/test_extractors.py
@@ -23,6 +23,9 @@ class TestExtractors(unittest.TestCase):
         config.set(("extractor", "nijie", "username"), email)
         config.set(("extractor", "seiga", "username"), email)
 
+    def tearDown(self):
+        config.clear()
+
     def _run_test(self, extr, url, result):
         content = "content" in result if result else False
         tjob = job.TestJob(url, content)