Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-26 12:42:29 +01:00
Code Issues Releases Wiki Activity

[warosu] fix extraction (#4634)

Browse Source
This commit is contained in:
Mike Fährmann 2023-10-13 23:03:39 +02:00
parent c4c4e4d2f4
commit 13ce3a9acb
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 74 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
gallery_dl/extractor/warosu.py
View File

@ -47,7 +47,7 @@ class WarosuThreadExtractor(Extractor):
def metadata(self, page): def metadata(self, page):
boardname = text.extr(page, "<title>", "</title>") boardname = text.extr(page, "<title>", "</title>")
title = text.extr(page, 'filetitle" itemprop="name">', '<') title = text.unescape(text.extr(page, "class=filetitle>", "<"))
return { return {
"board" : self.board, "board" : self.board,
"board_name": boardname.rpartition(" - ")[2], "board_name": boardname.rpartition(" - ")[2],
@ -57,8 +57,8 @@ class WarosuThreadExtractor(Extractor):
def posts(self, page): def posts(self, page):
"""Build a list of all post objects""" """Build a list of all post objects"""
page = text.extr(page, '<div class="content">', '<table>') page = text.extr(page, "<div class=content", "</form>")
needle = '<table itemscope itemtype="http://schema.org/Comment">' needle = "<table>"
return [self.parse(post) for post in page.split(needle)] return [self.parse(post) for post in page.split(needle)]
def parse(self, post): def parse(self, post):
@ -71,25 +71,23 @@ class WarosuThreadExtractor(Extractor):
data["ext"] = "." + data["extension"] data["ext"] = "." + data["extension"]
return data return data
@staticmethod def _extract_post(self, post):
def _extract_post(post):
extr = text.extract_from(post) extr = text.extract_from(post)
return { return {
"no" : extr('id="p', '"'), "no" : extr("id=p", ">"),
"name": extr('<span itemprop="name">', "</span>"), "name": extr("class=postername>", "<").strip(),
"time": extr('<span class="posttime" title="', '000">'), "time": extr("class=posttime title=", "000>"),
"now" : extr("", "<"), "now" : extr("", "<").strip(),
"com" : text.unescape(text.remove_html(extr( "com" : text.unescape(text.remove_html(extr(
'<blockquote><p itemprop="text">', '</p></blockquote>' "<blockquote>", "</blockquote>").strip())),
).strip())),
} }
@staticmethod def _extract_image(self, post, data):
def _extract_image(post, data):
extr = text.extract_from(post) extr = text.extract_from(post)
data["fsize"] = extr("<span> File: ", ", ") data["fsize"] = extr("<span> File: ", ", ")
data["w"] = extr("", "x") data["w"] = extr("", "x")
data["h"] = extr("", ", ") data["h"] = extr("", ", ")
data["filename"] = text.unquote(extr("", "<").rpartition(".")[0]) data["filename"] = text.unquote(extr(
extr("<br />", "") "", "<").rstrip().rpartition(".")[0])
data["image"] = "https:" + extr('<a href="', '"') extr("<br>", "")
data["image"] = self.root + extr("<a href=", ">")

62
test/results/warosu.py
View File

@ -12,17 +12,71 @@ __tests__ = (
"#url" : "https://warosu.org/jp/thread/16656025", "#url" : "https://warosu.org/jp/thread/16656025",
"#category": ("", "warosu", "thread"), "#category": ("", "warosu", "thread"),
"#class" : warosu.WarosuThreadExtractor, "#class" : warosu.WarosuThreadExtractor,
"#sha1_url" : "889d57246ed67e491e5b8f7f124e50ea7991e770", "#urls" : (
"#sha1_metadata": "c00ea4c5460c5986994f17bb8416826d42ca57c0", "https://warosu.org/data/jp/img/0166/56/1488487280004.png",
"https://warosu.org/data/jp/img/0166/56/1488493239417.png",
"https://warosu.org/data/jp/img/0166/56/1488493636725.jpg",
"https://warosu.org/data/jp/img/0166/56/1488493700040.jpg",
"https://warosu.org/data/jp/img/0166/56/1488499585168.jpg",
"https://warosu.org/data/jp/img/0166/56/1488530851199.jpg",
"https://warosu.org/data/jp/img/0166/56/1488536072155.jpg",
"https://warosu.org/data/jp/img/0166/56/1488603426484.png",
"https://warosu.org/data/jp/img/0166/56/1488647021253.jpg",
"https://warosu.org/data/jp/img/0166/56/1488866825031.jpg",
"https://warosu.org/data/jp/img/0166/56/1489094956868.jpg",
),
}, },
{ {
"#url" : "https://warosu.org/jp/thread/16658073", "#url" : "https://warosu.org/jp/thread/16658073",
"#category": ("", "warosu", "thread"), "#category": ("", "warosu", "thread"),
"#class" : warosu.WarosuThreadExtractor, "#class" : warosu.WarosuThreadExtractor,
"#sha1_url" : "4500cf3184b067424fd9883249bd543c905fbecd",
"#sha1_metadata": "7534edf4ec51891dbf44d775b73fbbefd52eec71",
"#sha1_content" : "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", "#sha1_content" : "d48df0a701e6599312bfff8674f4aa5d4fb8db1c",
"#urls" : "https://warosu.org/data/jp/img/0166/58/1488521824388.jpg",
"#count" : 1,
"board" : "jp",
"board_name": "Otaku Culture",
"com" : "Is this canon?",
"ext" : ".jpg",
"extension" : "jpg",
"filename" : "sadako-vs-kayako-movie-review",
"fsize" : "55 KB",
"h" : 675,
"image" : "https://warosu.org/data/jp/img/0166/58/1488521824388.jpg",
"name" : "Anonymous",
"no" : 16658073,
"now" : "Fri Mar 3 01:17:04 2017",
"thread" : "16658073",
"tim" : 1488521824388,
"time" : 1488503824,
"title" : "Is this canon?",
"w" : 450,
},
{
"#url" : "https://warosu.org/ic/thread/4604652",
"#category": ("", "warosu", "thread"),
"#class" : warosu.WarosuThreadExtractor,
"#pattern" : r"https://warosu\.org/data/ic/img/0046/04/1590\d{9}\.jpg",
"#count" : 133,
"board" : "ic",
"board_name": "Artwork/Critique",
"com" : str,
"ext" : ".jpg",
"filename" : str,
"fsize" : str,
"h" : range(200, 3507),
"image" : r"re:https://warosu\.org/data/ic/img/0046/04/1590\d+\.jpg",
"name" : "re:Anonymous|Dhe Specky Spider-Man",
"no" : range(4604652, 4620000),
"now" : r"re:\w\w\w \w\w\w \d\d \d\d:\d\d:\d\d 2020",
"thread" : "4604652",
"tim" : range(1590430159651, 1590755510488),
"time" : range(1590415759, 1590755510),
"title" : "American Classic Comic Artists",
"w" : range(200, 3000),
}, },
) )