[fanbox] add 'comments' option, extend 'metadata' option (#6287)

docs/configuration.rst

@@ -2312,6 +2312,19 @@ Description
     * ``"hitomi"``:  Download the corresponding gallery from ``hitomi.la``
 
 
+extractor.fanbox.comments
+-------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Extract ``comments`` metadata.
+
+    Note: This requires 1 or more additional API requests per post,
+    depending on the number of comments.
+
+
 extractor.fanbox.embeds
 -----------------------
 Type
@@ -2338,11 +2351,20 @@ Type
 Default
     ``false``
 Example
-    * ``user,plan``
-    * ``["user", "plan"]``
+    * ``user,plan,comments``
+    * ``["user", "plan", "comments"]``
 Description
     Extract ``plan`` and extended ``user`` metadata.
 
+    Supported fields when selecting which data to extract are
+
+    * ``comments``
+    * ``plan``
+    * ``user``
+
+    Note: ``comments`` can also be enabled via
+    `fanbox.comments <extractor.fanbox.comments_>`__
+
 
 extractor.flickr.access-token & .access-token-secret
 ----------------------------------------------------

gallery_dl/extractor/fanbox.py

@@ -29,7 +29,10 @@ class FanboxExtractor(Extractor):
     _warning = True
 
     def _init(self):
-        self.headers = {"Origin": self.root}
+        self.headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Origin": self.root,
+        }
         self.embeds = self.config("embeds", True)
 
         includes = self.config("metadata")
@@ -40,8 +43,12 @@ class FanboxExtractor(Extractor):
                 includes = ("user", "plan")
             self._meta_user = ("user" in includes)
             self._meta_plan = ("plan" in includes)
+            self._meta_comments = ("comments" in includes)
         else:
-            self._meta_user = self._meta_plan = False
+            self._meta_user = self._meta_plan = self._meta_comments = False
+
+        if self.config("comments"):
+            self._meta_comments = True
 
         if self._warning:
             if not self.cookies_check(("FANBOXSESSID",)):
@@ -124,6 +131,11 @@ class FanboxExtractor(Extractor):
                     plan = plans[0].copy()
                     plan["fee"] = fee
                 post["plan"] = plans[fee] = plan
+        if self._meta_comments:
+            if post["commentCount"]:
+                post["comments"] = list(self._get_comment_data(post_id))
+            else:
+                post["commentd"] = ()
 
         return content_body, post
 
@@ -160,6 +172,18 @@ class FanboxExtractor(Extractor):
 
         return plans
 
+    def _get_comment_data(self, post_id):
+        url = ("https://api.fanbox.cc/post.listComments"
+               "?limit=10&postId=" + post_id)
+
+        comments = []
+        while url:
+            url = text.ensure_http_scheme(url)
+            body = self.request(url, headers=self.headers).json()["body"]
+            comments.extend(body["items"])
+            url = body["nextUrl"]
+        return comments
+
     def _get_urls_from_post(self, content_body, post):
         num = 0
         cover_image = post.get("coverImageUrl")

test/results/fanbox.py

@@ -133,18 +133,27 @@ __tests__ = (
     "feeRequired": 300,
     "plan"       : {
         "creatorId": "saki9184",
-        "fee"      : 300,
-        "id"       : "430208",
-        "title"    : "ユーフォ、氷菓同人イラストR18",
+        "fee"      : 350,
+        "id"       : "414274",
+        "title"    : "入り浸りJKハルヒ",
     },
 },
 
 {
     "#url"     : "https://mochirong.fanbox.cc/posts/3746116",
-    "#comment" : "imageMap file order (#2718)",
+    "#comment" : "imageMap file order (#2718); comments metadata (#6287)",
     "#category": ("", "fanbox", "post"),
     "#class"   : fanbox.FanboxPostExtractor,
+    "#options" : {"metadata": "comments"},
     "#sha1_url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062",
+    "#urls"    : [
+        "https://pixiv.pximg.net/fanbox/public/images/post/3746116/cover/6h5w7F1MJWLeED6ODfHo6ZYQ.jpeg",
+        "https://downloads.fanbox.cc/images/post/3746116/ouTz7XZIeVD3FBOzoLhJ3ZTA.jpeg",
+        "https://downloads.fanbox.cc/images/post/3746116/hBs9bXEg6HvbqWT8QLD9g5ne.jpeg",
+        "https://downloads.fanbox.cc/images/post/3746116/C93E7db3C3sBqbDw6gQoZBMz.jpeg",
+    ],
+
+    "comments": "len:4",
 },
 
 {