mirror of
https://github.com/mikf/gallery-dl.git
synced 2025-01-31 19:51:34 +01:00
[bcy] reduce requests to '/item/detail/<id>' (#613)
The former implementation would try to use the embedded data from '/item/detail/' pages for every post, even if that wasn't really necessary. This commit also fixes some issues with posts only visible to logged in users.
This commit is contained in:
Mike Fährmann
parent
f33b13aacf
commit
3fb41c34c8
@ -31,96 +31,103 @@ class BcyExtractor(Extractor):
|
||||
iroot = "https://img-bcy-qn.pstatp.com"
|
||||
noop = self.config("noop")
|
||||
|
||||
for post_id in self.posts():
|
||||
post = self._parse_post(post_id)
|
||||
if not post:
|
||||
for post in self.posts():
|
||||
if not post["image_list"]:
|
||||
continue
|
||||
|
||||
yield Message.Directory, post
|
||||
for post["num"], image in enumerate(post["_multi"], 1):
|
||||
post["id"] = image["mid"]
|
||||
post["width"] = image["w"]
|
||||
post["height"] = image["h"]
|
||||
multi = None
|
||||
tags = post.get("post_tags") or ()
|
||||
data = {
|
||||
"user": {
|
||||
"id" : post["uid"],
|
||||
"name" : post["uname"],
|
||||
"avatar" : sub(iroot, post["avatar"].partition("~")[0]),
|
||||
},
|
||||
"post": {
|
||||
"id" : text.parse_int(post["item_id"]),
|
||||
"tags" : [t["tag_name"] for t in tags],
|
||||
"date" : text.parse_timestamp(post["ctime"]),
|
||||
"parody" : post["work"],
|
||||
"content": post["plain"],
|
||||
"likes" : post["like_count"],
|
||||
"shares" : post["share_count"],
|
||||
"replies": post["reply_count"],
|
||||
},
|
||||
}
|
||||
|
||||
yield Message.Directory, data
|
||||
for data["num"], image in enumerate(post["image_list"], 1):
|
||||
data["id"] = image["mid"]
|
||||
data["width"] = image["w"]
|
||||
data["height"] = image["h"]
|
||||
|
||||
url = image["path"].partition("~")[0]
|
||||
text.nameext_from_url(url, post)
|
||||
text.nameext_from_url(url, data)
|
||||
|
||||
if post["extension"]:
|
||||
if data["extension"]:
|
||||
if not url.startswith(iroot):
|
||||
url = sub(iroot, url)
|
||||
post["filter"] = ""
|
||||
yield Message.Url, url, post
|
||||
data["filter"] = ""
|
||||
yield Message.Url, url, data
|
||||
|
||||
else:
|
||||
post["filter"] = "watermark"
|
||||
yield Message.Url, image["origin"], post
|
||||
if not multi:
|
||||
if len(post["multi"]) < len(post["image_list"]):
|
||||
multi = self._data_from_post(post["item_id"])
|
||||
multi = multi["post_data"]["multi"]
|
||||
else:
|
||||
multi = post["multi"]
|
||||
image = multi[data["num"] - 1]
|
||||
|
||||
if image["origin"]:
|
||||
data["filter"] = "watermark"
|
||||
yield Message.Url, image["origin"], data
|
||||
|
||||
if noop:
|
||||
post["extension"] = ""
|
||||
post["filter"] = "noop"
|
||||
yield Message.Url, image["original_path"], post
|
||||
data["extension"] = ""
|
||||
data["filter"] = "noop"
|
||||
yield Message.Url, image["original_path"], data
|
||||
|
||||
def _parse_post(self, post_id):
|
||||
def posts(self):
|
||||
"""Returns an iterable with all relevant 'post' objects"""
|
||||
|
||||
def _data_from_post(self, post_id):
|
||||
url = "{}/item/detail/{}".format(self.root, post_id)
|
||||
response = self.request(url)
|
||||
if response.status_code >= 400:
|
||||
return None
|
||||
|
||||
data = json.loads(
|
||||
text.extract(response.text, 'JSON.parse("', '");')[0]
|
||||
page = self.request(url).text
|
||||
return json.loads(
|
||||
text.extract(page, 'JSON.parse("', '");')[0]
|
||||
.replace('\\\\u002F', '/')
|
||||
.replace('\\"', '"')
|
||||
)["detail"]
|
||||
|
||||
post = data["post_data"]
|
||||
if not post["multi"]:
|
||||
return None
|
||||
user = data["detail_user"]
|
||||
|
||||
return {
|
||||
"user": {
|
||||
"id" : user["uid"],
|
||||
"name" : user["uname"],
|
||||
"avatar" : user["avatar"],
|
||||
},
|
||||
"post": {
|
||||
"id" : text.parse_int(post["item_id"]),
|
||||
"tags" : [t["tag_name"] for t in post["post_tags"]],
|
||||
"date" : text.parse_timestamp(post["ctime"]),
|
||||
"parody" : text.parse_unicode_escapes(post["work"]),
|
||||
"content": post["plain"],
|
||||
"likes" : post["like_count"],
|
||||
"shares" : post["share_count"],
|
||||
"replies": post["reply_count"],
|
||||
},
|
||||
"_multi": post["multi"],
|
||||
}
|
||||
|
||||
|
||||
class BcyUserExtractor(BcyExtractor):
|
||||
"""Extractor for user timelines"""
|
||||
subcategory = "user"
|
||||
pattern = r"(?:https?://)?bcy\.net/u/(\d+)"
|
||||
test = ("https://bcy.net/u/1933712", {
|
||||
"pattern": r"https://img-bcy-qn.pstatp.com/\w+/\d+/post/\w+/\w+.jpg",
|
||||
"count": ">= 25",
|
||||
})
|
||||
test = (
|
||||
("https://bcy.net/u/1933712", {
|
||||
"pattern": r"https://img-bcy-qn.pstatp.com/\w+/\d+/post/\w+/.+jpg",
|
||||
"count": ">= 25",
|
||||
}),
|
||||
("https://bcy.net/u/109282764041", {
|
||||
"pattern": r"https://p\d-bcy.byteimg.com/img/banciyuan/[0-9a-f]+"
|
||||
r"~tplv-banciyuan-logo-v3:.+\.image",
|
||||
"range": "1-25",
|
||||
"count": 25,
|
||||
}),
|
||||
)
|
||||
|
||||
def posts(self):
|
||||
url = self.root + "/apiv3/user/selfPosts"
|
||||
params = {
|
||||
"uid": self.item_id,
|
||||
"since": None,
|
||||
# "_signature": None,
|
||||
}
|
||||
params = {"uid": self.item_id, "since": None}
|
||||
|
||||
while True:
|
||||
data = self.request(url, params=params).json()
|
||||
|
||||
item = None
|
||||
for item in data["data"]["items"]:
|
||||
if item["item_detail"]["multi"]:
|
||||
yield item["item_detail"]["item_id"]
|
||||
yield item["item_detail"]
|
||||
|
||||
if not item:
|
||||
return
|
||||
@ -131,33 +138,51 @@ class BcyPostExtractor(BcyExtractor):
|
||||
"""Extractor for individual posts"""
|
||||
subcategory = "post"
|
||||
pattern = r"(?:https?://)?bcy\.net/item/detail/(\d+)"
|
||||
test = ("https://bcy.net/item/detail/6355835481002893070", {
|
||||
"url": "301202375e61fd6e0e2e35de6c3ac9f74885dec3",
|
||||
"count": 1,
|
||||
"keyword": {
|
||||
"user": {
|
||||
"id" : 1933712,
|
||||
"name" : "wukloo",
|
||||
"avatar" : str,
|
||||
test = (
|
||||
("https://bcy.net/item/detail/6355835481002893070", {
|
||||
"url": "301202375e61fd6e0e2e35de6c3ac9f74885dec3",
|
||||
"count": 1,
|
||||
"keyword": {
|
||||
"user": {
|
||||
"id" : 1933712,
|
||||
"name" : "wukloo",
|
||||
"avatar" : "re:https://img-bcy-qn.pstatp.com/Public/",
|
||||
},
|
||||
"post": {
|
||||
"id" : 6355835481002893070,
|
||||
"tags" : list,
|
||||
"date" : "dt:2016-11-22 08:47:46",
|
||||
"parody" : "东方PROJECT",
|
||||
"content": "re:根据微博的建议稍微做了点修改",
|
||||
"likes" : int,
|
||||
"shares" : int,
|
||||
"replies": int,
|
||||
},
|
||||
"id": 8330182,
|
||||
"num": 1,
|
||||
"width" : 3000,
|
||||
"height": 1687,
|
||||
"filename": "712e0780b09011e696f973c3d1568337",
|
||||
"extension": "jpg",
|
||||
},
|
||||
"post": {
|
||||
"id" : 6355835481002893070,
|
||||
"tags" : list,
|
||||
"date" : "dt:2016-11-22 08:47:46",
|
||||
"parody" : "东方PROJECT",
|
||||
"content": "re:根据微博的建议稍微做了点修改",
|
||||
"likes" : int,
|
||||
"shares" : int,
|
||||
"replies": int,
|
||||
},
|
||||
"id": 8330182,
|
||||
"num": 1,
|
||||
"width" : 3000,
|
||||
"height": 1687,
|
||||
"filename": "712e0780b09011e696f973c3d1568337",
|
||||
"extension": "jpg",
|
||||
},
|
||||
})
|
||||
}),
|
||||
# only watermarked images available
|
||||
("https://bcy.net/item/detail/6780546160802143236", {
|
||||
"pattern": r"https://p\d-bcy.byteimg.com/img/banciyuan/[0-9a-f]+"
|
||||
r"~tplv-banciyuan-logo-v3:.+\.image",
|
||||
"count": 8,
|
||||
"keyword": {"filter": "watermark"}
|
||||
}),
|
||||
# only visible to logged in users
|
||||
("https://bcy.net/item/detail/6747523535150783495", {
|
||||
"count": 0,
|
||||
}),
|
||||
)
|
||||
|
||||
def posts(self):
|
||||
return (self.item_id,)
|
||||
data = self._data_from_post(self.item_id)
|
||||
post = data["post_data"]
|
||||
post["image_list"] = post["multi"]
|
||||
post["plain"] = text.parse_unicode_escapes(post["plain"])
|
||||
post.update(data["detail_user"])
|
||||
return (post,)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user