mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-25 04:02:32 +01:00
[furaffinity] add 'external' option (closes #1492)
This commit is contained in:
parent
b5b1cf22b7
commit
5b1c62bfa9
@ -1128,6 +1128,16 @@ Description
|
||||
* ``"html"``: Raw HTML content
|
||||
|
||||
|
||||
extractor.furaffinity.external
|
||||
------------------------------
|
||||
Type
|
||||
``bool``
|
||||
Default
|
||||
``false``
|
||||
Description
|
||||
Follow external URLs linked in descriptions.
|
||||
|
||||
|
||||
extractor.furaffinity.include
|
||||
-----------------------------
|
||||
Type
|
||||
|
@ -98,6 +98,7 @@
|
||||
"furaffinity":
|
||||
{
|
||||
"descriptions": "text",
|
||||
"external": false,
|
||||
"include": "gallery"
|
||||
},
|
||||
"gfycat":
|
||||
|
@ -29,9 +29,10 @@ class FuraffinityExtractor(Extractor):
|
||||
self.offset = 0
|
||||
|
||||
if self.config("descriptions") == "html":
|
||||
self._process_description = lambda x: x.strip()
|
||||
self._process_description = str.strip
|
||||
|
||||
def items(self):
|
||||
external = self.config("external", False)
|
||||
metadata = self.metadata()
|
||||
for post_id in util.advance(self.posts(), self.offset):
|
||||
post = self._parse_post(post_id)
|
||||
@ -41,6 +42,11 @@ class FuraffinityExtractor(Extractor):
|
||||
yield Message.Directory, post
|
||||
yield Message.Url, post["url"], post
|
||||
|
||||
if external:
|
||||
for url in text.extract_iter(
|
||||
post["_description"], 'href="http', '"'):
|
||||
yield Message.Queue, "http" + url, post
|
||||
|
||||
def posts(self):
|
||||
return self._pagination()
|
||||
|
||||
@ -80,8 +86,7 @@ class FuraffinityExtractor(Extractor):
|
||||
data["tags"] = text.split_html(tags)
|
||||
data["title"] = text.unescape(extr("<h2><p>", "</p></h2>"))
|
||||
data["artist"] = extr("<strong>", "<")
|
||||
data["description"] = self._process_description(extr(
|
||||
'class="section-body">', '</div>'))
|
||||
data["_description"] = extr('class="section-body">', '</div>')
|
||||
data["views"] = pi(rh(extr('class="views">', '</span>')))
|
||||
data["favorites"] = pi(rh(extr('class="favorites">', '</span>')))
|
||||
data["comments"] = pi(rh(extr('class="comments">', '</span>')))
|
||||
@ -108,12 +113,12 @@ class FuraffinityExtractor(Extractor):
|
||||
data["tags"] = text.split_html(extr(
|
||||
'id="keywords">', '</div>'))[::2]
|
||||
data["rating"] = extr('<img alt="', ' ')
|
||||
data["description"] = self._process_description(extr(
|
||||
"</table>", "</table>"))
|
||||
data["_description"] = extr("</table>", "</table>")
|
||||
|
||||
data["artist_url"] = data["artist"].replace("_", "").lower()
|
||||
data["user"] = self.user or data["artist_url"]
|
||||
data["date"] = text.parse_timestamp(data["filename"].partition(".")[0])
|
||||
data["description"] = self._process_description(data["_description"])
|
||||
|
||||
return data
|
||||
|
||||
@ -273,6 +278,13 @@ class FuraffinityPostExtractor(FuraffinityExtractor):
|
||||
"height" : 120,
|
||||
},
|
||||
}),
|
||||
# 'external' option (#1492)
|
||||
("https://www.furaffinity.net/view/42166511/", {
|
||||
"options": (("external", True),),
|
||||
"pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/"
|
||||
r"|http://www\.postybirb\.com",
|
||||
"count": 2,
|
||||
}),
|
||||
("https://furaffinity.net/view/21835115/"),
|
||||
("https://sfw.furaffinity.net/view/21835115/"),
|
||||
("https://www.furaffinity.net/full/21835115/"),
|
||||
|
Loading…
Reference in New Issue
Block a user