[hentaifoundry] add extractor

gallery_dl/extractor/__init__.py

@@ -21,6 +21,7 @@ modules = [
     "danbooru",
     "deviantart",
     "e621",
+    "hentaifoundry",
     "hitomi",
     "imagebam",
     "imgbox",

gallery_dl/extractor/hentaifoundry.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extract images from http://www.hentai-foundry.com/"""
+
+from .common import Extractor, Message
+from .. import text
+import os.path
+
+info = {
+    "category": "hentaifoundry",
+    "extractor": "HentaiFoundryExtractor",
+    "directory": ["{category}", "{artist}"],
+    "filename": "{category}_{index}_{title}.{extension}",
+    "pattern": [
+        r"(?:https?://)?(?:www\.)?hentai-foundry\.com/pictures/user/([^/]+)",
+        r"(?:https?://)?(?:www\.)?hentai-foundry\.com/user/([^/]+)/profile",
+    ],
+}
+
+class HentaiFoundryExtractor(Extractor):
+
+    url_base = "http://www.hentai-foundry.com/pictures/user/"
+
+    def __init__(self, match):
+        Extractor.__init__(self)
+        self.artist = match.group(1)
+
+    def items(self):
+        data, token = self.get_job_metadata()
+        self.set_filters(token)
+        yield Message.Version, 1
+        yield Message.Directory, data
+        for image in self.get_images():
+            image.update(data)
+            yield Message.Url, image["url"], image
+
+    def get_images(self):
+        num = 1
+        while True:
+            pos = 0
+            url = self.url_base + self.artist + "/page/" + str(num)
+            page = self.request(url).text
+            for i in range(25):
+                part, pos = text.extract(page, 'thumbTitle"><a href="/pictures/user/', '"', pos)
+                if not part:
+                    return
+                yield self.get_image_metadata(self.url_base + part)
+            num += 1
+
+    def get_job_metadata(self):
+        """Collect metadata for extractor-job"""
+        page = self.request(self.url_base + self.artist + "?enterAgree=1").text
+        token, pos = text.extract(page, 'hidden" value="', '"')
+        count, pos = text.extract(page, 'class="active" >Pictures (', ')', pos)
+        return {
+            "category": info["category"],
+            "artist": self.artist,
+            "count": count,
+        }, token
+
+    def get_image_metadata(self, url):
+        """Collect metadata for an image"""
+        page = self.request(url).text
+        index = text.extract(url, '/', '/', len(self.url_base) + len(self.artist))[0]
+        title, pos = text.extract(page, 'Pictures</a> &raquo; <span>', '<')
+        url  , pos = text.extract(page, '//pictures.hentai-foundry.com', '"', pos)
+        return {
+            "url": "http://pictures.hentai-foundry.com" + url,
+            "index": index,
+            "title": text.unescape(title),
+            "extension": os.path.splitext(url)[1][1:],
+        }
+
+    def set_filters(self, token):
+        formdata = {
+            "YII_CSRF_TOKEN": token,
+            "rating_nudity": 3,
+            "rating_violence": 3,
+            "rating_profanity": 3,
+            "rating_racism": 3,
+            "rating_sex": 3,
+            "rating_spoilers": 3,
+            "rating_yaoi": 1,
+            "rating_yuri": 1,
+            "rating_teen": 1,
+            "rating_guro": 1,
+            "rating_furry": 1,
+            "rating_beast": 1,
+            "rating_male": 1,
+            "rating_female": 1,
+            "rating_futa": 1,
+            "rating_other": 1,
+            "filter_media": "A",
+            "filter_order": "date_new",
+            "filter_type": 0,
+        }
+        self.request("http://www.hentai-foundry.com/site/filters",
+            method="post", data=formdata)
+
+# enterAgree=1
+
+#YII_CSRF_TOKEN=388b659daeab4517ea5ca6d93b9253d4c59a12df