Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2025-01-31 11:41:35 +01:00
Code Issues Releases Wiki Activity

[poringa] improvements and fixes

Browse Source 
- add 'num' and 'count' metadata fields
- prevent crash for "private" posts
- prevent crash when there's no 'main-info'
- update tests
This commit is contained in:
Mike Fährmann 2023-12-30 20:37:09 +01:00
parent e348da7a06
commit 9f21c839ad
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 45 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
gallery_dl/extractor/poringa.py
View File

@ -17,8 +17,8 @@ BASE_PATTERN = r"(?:https?://)?(?:www\.)?poringa\.net"
class PoringaExtractor(Extractor):
category = "poringa"
directory_fmt = ("{category}", "{user}", "{post_id}")
filename_fmt = "{post_id}_{title}_{filename}.{extension}"
archive_fmt = "{post_id}"
filename_fmt = "{post_id}_{title}_{num:>03}_{filename}.{extension}"
archive_fmt = "{post_id}_{num}"
root = "http://www.poringa.net"
def __init__(self, match):
@ -31,36 +31,45 @@ class PoringaExtractor(Extractor):
url = "{}/posts/imagenes/{}".format(self.root, post_id)
try:
page = self.request(url).text
response = self.request(url)
except exception.HttpError as exc:
self.log.warning(
"Unable to fetch posts for '%s' (%s)", post_id, exc)
continue
if "/registro-login?" in response.url:
self.log.warning("Private post '%s'", post_id)
continue
page = response.text
title, pos = text.extract(
page, 'property="og:title" content="', '"')
pos = page.index('<div class="main-info', pos)
user, pos = text.extract(
page, 'href="http://www.poringa.net/', '"', pos)
try:
pos = page.index('<div class="main-info', pos)
user, pos = text.extract(
page, 'href="http://www.poringa.net/', '"', pos)
except ValueError:
user = None
if not user:
user = "poringa"
data = {
"post_id" : post_id,
"post_id" : post_id,
"title" : text.unescape(title),
"user" : text.unquote(user),
"_http_headers": {"Referer": url},
}
yield Message.Directory, data
main_post = text.extr(
page, 'property="dc:content" role="main">', '</div>')
for url in text.extract_iter(
main_post,
'<img class="imagen" border="0" src="',
'"',
):
urls = list(text.extract_iter(
main_post, '<img class="imagen" border="0" src="', '"'))
data["count"] = len(urls)
yield Message.Directory, data
for data["num"], url in enumerate(urls, 1):
yield Message.Url, url, text.nameext_from_url(url, data)
def posts(self):
@ -95,8 +104,8 @@ class PoringaExtractor(Extractor):
class PoringaPostExtractor(PoringaExtractor):
"""Extractor for posts on poringa.net"""
subcategory = "post"
pattern = BASE_PATTERN + r"/posts/imagenes/(\d+)/[a-zA-Z0-9_-]+\.html"
example = "http://www.poringa.net/posts/imagenes/12/TITLE.html"
pattern = BASE_PATTERN + r"/posts/imagenes/(\d+)"
example = "http://www.poringa.net/posts/imagenes/12345/TITLE.html"
def posts(self):
return (self.item,)
@ -104,12 +113,12 @@ class PoringaPostExtractor(PoringaExtractor):
class PoringaUserExtractor(PoringaExtractor):
subcategory = "user"
pattern = BASE_PATTERN + r"/([a-zA-Z0-9_-]+)$"
pattern = BASE_PATTERN + r"/(\w+)$"
example = "http://www.poringa.net/USER"
def posts(self):
url = "{}/buscar/".format(self.root)
params = {"q": text.unquote(self.item)}
url = self.root + "/buscar/"
params = {"q": self.item}
return self._pagination(url, params)
@ -120,7 +129,7 @@ class PoringaSearchExtractor(PoringaExtractor):
def posts(self):
url = self.root + "/buscar/"
params = {"q": text.unquote(self.item)}
params = {"q": self.item}
return self._pagination(url, params)

27
test/results/poringa.py
View File

@ -12,36 +12,43 @@ __tests__ = (
"#url" : "http://www.poringa.net/posts/imagenes/3051081/Turrita-alto-ojete.html",
"#category": ("", "poringa", "post"),
"#class" : poringa.PoringaPostExtractor,
"#pattern" : r"http://www\.poringa\.net/posts/imagenes/3051081/[a-zA-Z0-9_-]+\.html",
"#count" : 26,
"post_id" : "3051081",
"title" : "turrita alto ojete...",
"user" : "vipower1top",
"count" : 26,
"num" : range(1, 26),
"post_id" : "3051081",
"title" : "turrita alto ojete...",
"user" : "vipower1top",
},
{
"#url" : "http://www.poringa.net/posts/imagenes/3095554/Otra-culona-de-instagram.html",
"#category": ("", "poringa", "post"),
"#class" : poringa.PoringaPostExtractor,
"#pattern" : r"http://www\.poringa\.net/posts/imagenes/3095554/[a-zA-Z0-9_-]+\.html",
"#count" : 15,
"post_id" : "3095554",
"title" : "Otra culona de instagram",
"user" : "Expectro007",
"count" : 15,
"num" : range(1, 15),
"post_id" : "3095554",
"title" : "Otra culona de instagram",
"user" : "Expectro007",
},
{
"#url" : "http://www.poringa.net/Expectro007",
"#category": ("", "poringa", "user"),
"#class" : poringa.PoringaUserExtractor,
"#pattern" : r"https?://img-[0-9]\.poringa\.net/poringa/img/[a-zA-Z0-9/{2}]{12}[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+\.jpg",
"#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././Expectro007/\w{3}\.(jpg|png|gif)",
"#count" : range(500, 600),
},
{
"#url" : "http://www.poringa.net/buscar/?&q=yuslopez",
"#category": ("", "poringa", "search"),
"#class" : poringa.PoringaSearchExtractor,
"#pattern" : r"https?://img-[0-9]\.poringa\.net/poringa/img/[a-zA-Z0-9/{2}]{12}[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+\.jpg",
"#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././\w+/\w{3}\.(jpg|png|gif)",
"#range" : "1-50",
"#count" : 50,
},
)