', '
')
+ for url in text.extract_iter(
+ main_post,
+ '"
+ page,
+ '
"
)
chapter += r"" # so we can match the date
url, pos = text.extract(chapter, '"
+ chapter,
+ '"
)
title = text.extr(title, r"", r"")
if title is None or title == "" or title == "":
From f6ce870885a1df8dfed788c0c9c2cadee1c21f8f Mon Sep 17 00:00:00 2001
From: bug-assassin <7788433+bug-assassin@users.noreply.github.com>
Date: Thu, 28 Dec 2023 17:25:15 -0500
Subject: [PATCH 10/77] Better variable names
---
gallery_dl/extractor/bato.py | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/gallery_dl/extractor/bato.py b/gallery_dl/extractor/bato.py
index 082c5e0a..d29a58bf 100644
--- a/gallery_dl/extractor/bato.py
+++ b/gallery_dl/extractor/bato.py
@@ -97,32 +97,33 @@ class BatoMangaExtractor(BatoBase, MangaExtractor):
results = []
for chapter_num in range(num_chapters):
- chapter = text.extr(
+ chapter_info = text.extr(
page,
'
"
)
- chapter += r"" # so we can match the date
- url, pos = text.extract(chapter, '" # so we can match the date
+ url, pos = text.extract(chapter_info, '"
)
title = text.extr(title, r"", r"")
if title is None or title == "" or title == "":
- title, _ = text.extract(chapter, ">", "", pos)
+ title, _ = text.extract(chapter_info, ">", "", pos)
- date = text.extr(chapter, "")
+ date = text.extr(chapter_info, "")
date = text.extr(date, 'time="', '"')
data["date"] = date
data["title"] = title
- data["chapter"] = text.parse_int(chapter_major)
- data["chapter_minor"] = sep + chapter_minor
+ data["chapter"] = text.parse_int(chapt_major)
+ data["chapter_minor"] = sep + chapt_minor
if url.startswith("/"):
url = self.root + url
From 085411f3f13d691f283f1d3fcfb99d80bbb19b29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Sat, 30 Dec 2023 16:07:56 +0100
Subject: [PATCH 11/77] [rule34] recognize URLs with 'www' subdomain (#4984)
---
gallery_dl/extractor/gelbooru_v02.py | 2 +-
test/results/rule34.py | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/gallery_dl/extractor/gelbooru_v02.py b/gallery_dl/extractor/gelbooru_v02.py
index 0864b9f6..0c8af3d5 100644
--- a/gallery_dl/extractor/gelbooru_v02.py
+++ b/gallery_dl/extractor/gelbooru_v02.py
@@ -168,7 +168,7 @@ INSTANCES = {
},
"rule34": {
"root": "https://rule34.xxx",
- "pattern": r"rule34\.xxx",
+ "pattern": r"(?:www\.)?rule34\.xxx",
"api_root": "https://api.rule34.xxx",
},
"safebooru": {
diff --git a/test/results/rule34.py b/test/results/rule34.py
index ca90e511..f8fefa32 100644
--- a/test/results/rule34.py
+++ b/test/results/rule34.py
@@ -34,6 +34,13 @@ __tests__ = (
"#count" : 3,
},
+{
+ "#url" : "https://www.rule34.xxx/index.php?page=post&s=view&id=863",
+ "#comment" : "www subdomain",
+ "#category": ("gelbooru_v02", "rule34", "post"),
+ "#class" : gelbooru_v02.GelbooruV02PostExtractor,
+},
+
{
"#url" : "https://rule34.xxx/index.php?page=post&s=view&id=863",
"#category": ("gelbooru_v02", "rule34", "post"),
From caceb14fc2802237f67eb2b70b31d2c34ec055a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Sat, 30 Dec 2023 17:26:57 +0100
Subject: [PATCH 12/77] [tests] fail when a results file contains syntax errors
or is otherwise not importable
---
test/results/__init__.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/test/results/__init__.py b/test/results/__init__.py
index 0fe87462..c54bea56 100644
--- a/test/results/__init__.py
+++ b/test/results/__init__.py
@@ -13,12 +13,8 @@ __directory__ = os.path.dirname(__file__)
@functools.lru_cache(maxsize=None)
def tests(name):
- try:
- module = __import__(name, globals(), None, (), 1)
- return module.__tests__
- except Exception as exc:
- print(exc)
- return ()
+ module = __import__(name, globals(), None, (), 1)
+ return module.__tests__
def all():
From 00d83d9588c3fa9ed6b753d0c6baa2dc90ce4a5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Sat, 30 Dec 2023 18:33:46 +0100
Subject: [PATCH 13/77] [rule34us] add fallback for 'video-cdn1' videos (#4985)
---
gallery_dl/extractor/rule34us.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/gallery_dl/extractor/rule34us.py b/gallery_dl/extractor/rule34us.py
index 6439a225..cf70cccb 100644
--- a/gallery_dl/extractor/rule34us.py
+++ b/gallery_dl/extractor/rule34us.py
@@ -38,7 +38,11 @@ class Rule34usExtractor(BooruExtractor):
"height" : extr(' x ', 'h'),
"file_url": extr(' src="', '"'),
}
- post["md5"] = post["file_url"].rpartition("/")[2].partition(".")[0]
+
+ url = post["file_url"]
+ if "//video-cdn1." in url:
+ post["_fallback"] = (url.replace("//video-cdn1.", "//video."),)
+ post["md5"] = url.rpartition("/")[2].partition(".")[0]
tags = collections.defaultdict(list)
for tag_type, tag_name in self._find_tags(page):
From 9f21c839ad6f0312fea3868568cdcb3313d09a94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Sat, 30 Dec 2023 20:37:09 +0100
Subject: [PATCH 14/77] [poringa] improvements and fixes
- add 'num' and 'count' metadata fields
- prevent crash for "private" posts
- prevent crash when there's no 'main-info'
- update tests
---
gallery_dl/extractor/poringa.py | 47 ++++++++++++++++++++-------------
test/results/poringa.py | 27 ++++++++++++-------
2 files changed, 45 insertions(+), 29 deletions(-)
diff --git a/gallery_dl/extractor/poringa.py b/gallery_dl/extractor/poringa.py
index e5e80d57..0149d060 100644
--- a/gallery_dl/extractor/poringa.py
+++ b/gallery_dl/extractor/poringa.py
@@ -17,8 +17,8 @@ BASE_PATTERN = r"(?:https?://)?(?:www\.)?poringa\.net"
class PoringaExtractor(Extractor):
category = "poringa"
directory_fmt = ("{category}", "{user}", "{post_id}")
- filename_fmt = "{post_id}_{title}_{filename}.{extension}"
- archive_fmt = "{post_id}"
+ filename_fmt = "{post_id}_{title}_{num:>03}_{filename}.{extension}"
+ archive_fmt = "{post_id}_{num}"
root = "http://www.poringa.net"
def __init__(self, match):
@@ -31,36 +31,45 @@ class PoringaExtractor(Extractor):
url = "{}/posts/imagenes/{}".format(self.root, post_id)
try:
- page = self.request(url).text
+ response = self.request(url)
except exception.HttpError as exc:
self.log.warning(
"Unable to fetch posts for '%s' (%s)", post_id, exc)
continue
+ if "/registro-login?" in response.url:
+ self.log.warning("Private post '%s'", post_id)
+ continue
+
+ page = response.text
title, pos = text.extract(
page, 'property="og:title" content="', '"')
- pos = page.index('
Date: Sat, 30 Dec 2023 22:25:59 +0100
Subject: [PATCH 15/77] [nijie] add 'count' metadata field
https://github.com/mikf/gallery-dl/issues/146#issuecomment-1812849102
---
gallery_dl/extractor/nijie.py | 7 +++++--
test/results/horne.py | 3 +++
test/results/nijie.py | 7 +++++--
3 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py
index 57c31184..b9917057 100644
--- a/gallery_dl/extractor/nijie.py
+++ b/gallery_dl/extractor/nijie.py
@@ -55,9 +55,12 @@ class NijieExtractor(AsynchronousMixin, BaseExtractor):
else:
data["user_id"] = data["artist_id"]
data["user_name"] = data["artist_name"]
- yield Message.Directory, data
- for num, url in enumerate(self._extract_images(image_id, page)):
+ urls = list(self._extract_images(image_id, page))
+ data["count"] = len(urls)
+
+ yield Message.Directory, data
+ for num, url in enumerate(urls):
image = text.nameext_from_url(url, {
"num": num,
"url": "https:" + url,
diff --git a/test/results/horne.py b/test/results/horne.py
index 9058a481..f6bddba8 100644
--- a/test/results/horne.py
+++ b/test/results/horne.py
@@ -83,6 +83,7 @@ __tests__ = (
"artist_id" : 58000,
"artist_name": "のえるわ",
+ "count" : 1,
"date" : "dt:2018-01-29 14:25:39",
"description": "前回とシチュがまるかぶり \r\n竿野郎は塗るのだるかった",
"extension" : "png",
@@ -113,9 +114,11 @@ __tests__ = (
"artist_id" : 58000,
"artist_name": "のえるわ",
+ "count" : 4,
"date" : "dt:2018-02-04 14:47:24",
"description": "ノエル「そんなことしなくても、言ってくれたら咥えるのに・・・♡」",
"image_id" : 8716,
+ "num" : range(0, 3),
"tags" : [
"男の娘",
"フェラ",
diff --git a/test/results/nijie.py b/test/results/nijie.py
index 01ac8fac..a2c05c81 100644
--- a/test/results/nijie.py
+++ b/test/results/nijie.py
@@ -31,12 +31,13 @@ __tests__ = (
"artist_id" : 44,
"artist_name": "ED",
+ "count" : 1,
"date" : datetime.datetime,
"description": str,
"extension" : "jpg",
"filename" : str,
"image_id" : int,
- "num" : int,
+ "num" : 0,
"tags" : list,
"title" : str,
"url" : r"re:https://pic.nijie.net/\d+/nijie/.*jpg$",
@@ -102,11 +103,12 @@ __tests__ = (
"#class" : nijie.NijieImageExtractor,
"#urls" : "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg",
"#sha1_url" : "3d654e890212ba823c9647754767336aebc0a743",
- "#sha1_metadata": "41da5d0e178b04f01fe72460185df52fadc3c91b",
+ "#sha1_metadata": "58e716bcb03b431cae901178c198c787908e1c0c",
"#sha1_content" : "d85e3ea896ed5e4da0bca2390ad310a4df716ca6",
"artist_id" : 44,
"artist_name": "ED",
+ "count" : 1,
"date" : "dt:2014-01-18 19:58:21",
"description": "租絵にてお邪魔いたし候\r\n是非ともこの”おっぱい”をご高覧賜りたく馳せ参じた次第\r\n長文にて失礼仕る\r\n\r\nまず全景でありますが、首を右に傾けてみて頂きたい\r\nこの絵図は茶碗を眺めていた私が思わぬ美しさにて昇天したときのものを、筆をとり、したためたものである（トレースではない）\r\n筆は疾風の如く走り、半刻過ぎには私好みの”おっぱい”になっていたのである！\r\n次に細部をみて頂きたい\r\n絵図を正面から見直して頂くと、なんとはんなりと美しいお椀型をしたおっぱいであろうか 右手から緩やかに生まれる曲線は左手に進むにつれて、穏やかな歪みを含み流れる これは所謂轆轤目であるが三重の紐でおっぱいをぐるぐると巻きつけた情景そのままであり、この歪みから茶碗の均整は崩れ、たぷんたぷんのおっぱいの重量感を醸し出している！\r\nさらに左手に進めば梅花皮（カイラギ）を孕んだ高大が現れる 今回は点線にて表現するが、その姿は乳首から母乳が噴出するが如く 或は精子をぶっかけられたが如く 白くとろっとした釉薬の凝固が素晴しい景色をつくりだしているのである！\r\n最後には極めつけ、すくっと螺旋を帯びながらそそり立つ兜巾（ときん）！この情景はまさしく乳首である！ 全体をふんわりと盛り上げさせる乳輪にちょこっと存在する乳頭はぺろりと舌で確かめ勃起させたくなる風情がある！\r\n\r\nこれを”おっぱい”と呼ばずなんと呼ぼうや！？\r\n\r\n興奮のあまり失礼致した\r\n御免",
"extension" : "jpg",
@@ -133,6 +135,7 @@ __tests__ = (
"artist_id" : 49509,
"artist_name": "黒川 竜",
+ "count" : 4,
"date" : "dt:2023-12-02 04:19:29",
"description": "【DLサイトコム】ウィンターセール 30%OFF\r\n期間：2024年2月14日まで\r\n【toloveるドリンク】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ042727.html\r\n【toloveるドリンク２】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043289.html\r\n【クランクランBIG】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043564.html",
"image_id" : 594044,
From fe2147b3efe580f5b44bbab30f4d2ef30be7ea92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Sun, 31 Dec 2023 01:24:12 +0100
Subject: [PATCH 16/77] [docs] document 'write-pages' (#4543)
---
docs/configuration.rst | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 2a9029ed..de180973 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -1066,6 +1066,25 @@ Description
after a colon ``:``, for example ``{date:%Y%m%d}``.
+extractor.*.write-pages
+-----------------------
+Type
+ * ``bool``
+ * ``string``
+Default
+ ``false``
+Description
+ During data extraction,
+ write received HTTP request data
+ to enumerated files in the current working directory.
+
+ Special values:
+
+ * ``"all"``: Include HTTP request and response headers. Hide ``Authorization``, ``Cookie``, and ``Set-Cookie`` values.
+ * ``"ALL"``: Include all HTTP request and response headers.
+
+
+
Extractor-specific Options
==========================
From 27d5fc3697f31f50a1a78df5224d42e8b43e9c53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Mon, 1 Jan 2024 16:08:10 +0100
Subject: [PATCH 17/77] [docs] document 'tls12' (#4543)
https://github.com/mikf/gallery-dl/issues/4760#issuecomment-1793345940
---
docs/configuration.rst | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/docs/configuration.rst b/docs/configuration.rst
index de180973..e922f813 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -627,6 +627,20 @@ Description
`ssl.SSLContext.set_ciphers() `__
+extractor.*.tls12
+-----------------
+Type
+ ``bool``
+Default
+ * ``true``
+ * ``false`` for ``patreon``, ``pixiv:series``
+Description
+ Allow selecting TLS 1.2 cipher suites.
+
+ Can be disabled to alter TLS fingerprints
+ and potentially bypass Cloudflare blocks.
+
+
extractor.*.keywords
--------------------
Type
From 7aa1c9671baf6193cadc878c6a44477e8575dbc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Mon, 1 Jan 2024 02:51:34 +0100
Subject: [PATCH 18/77] [tests] fix 'invalid escape sequence' warnings
---
test/results/4plebs.py | 2 +-
test/results/imgbb.py | 2 +-
test/results/paheal.py | 2 +-
test/results/raddle.py | 4 ++--
test/results/wikiart.py | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/test/results/4plebs.py b/test/results/4plebs.py
index bae62608..affe14d8 100644
--- a/test/results/4plebs.py
+++ b/test/results/4plebs.py
@@ -12,7 +12,7 @@ __tests__ = (
"#url" : "https://archive.4plebs.org/tg/thread/54059290",
"#category": ("foolfuuka", "4plebs", "thread"),
"#class" : foolfuuka.FoolfuukaThreadExtractor,
- "#pattern" : "https://i\.4pcdn\.org/tg/1[34]\d{11}\.(jpg|png|gif)",
+ "#pattern" : r"https://i\.4pcdn\.org/tg/1[34]\d{11}\.(jpg|png|gif)",
"#count" : 30,
},
diff --git a/test/results/imgbb.py b/test/results/imgbb.py
index b2351d0f..e2d1bc33 100644
--- a/test/results/imgbb.py
+++ b/test/results/imgbb.py
@@ -21,7 +21,7 @@ __tests__ = (
"album_id" : "i5PggF",
"album_name" : "British Scrap Book",
"extension" : "jpg",
- "id" : "re:^\w{7}$",
+ "id" : r"re:^\w{7}$",
"title" : str,
"url" : r"re:https://i\.ibb\.co/\w{7}/[\w-]+\.jpg",
"user" : "folkie",
diff --git a/test/results/paheal.py b/test/results/paheal.py
index 1772593b..46b210f6 100644
--- a/test/results/paheal.py
+++ b/test/results/paheal.py
@@ -12,7 +12,7 @@ __tests__ = (
"#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1",
"#category": ("shimmie2", "paheal", "tag"),
"#class" : paheal.PahealTagExtractor,
- "#pattern" : "https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20|https://r34i\.paheal-cdn\.net/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}$",
+ "#pattern" : r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20|https://r34i\.paheal-cdn\.net/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}$",
"#count" : range(70, 200),
"date" : "type:datetime",
diff --git a/test/results/raddle.py b/test/results/raddle.py
index 4e60abb7..0c9de429 100644
--- a/test/results/raddle.py
+++ b/test/results/raddle.py
@@ -21,7 +21,7 @@ __tests__ = (
"#category": ("postmill", "raddle.me", "forum"),
"#class" : postmill.PostmillForumExtractor,
"#count" : 1,
- "#pattern" : "^https://raddle\.me/f/traa/156646/click-here-to-go-to-f-traaaaaaannnnnnnnnns$",
+ "#pattern" : r"^https://raddle\.me/f/traa/156646/click-here-to-go-to-f-traaaaaaannnnnnnnnns$",
},
{
@@ -97,7 +97,7 @@ __tests__ = (
"#comment" : "Link + text post (with text disabled)",
"#category": ("postmill", "raddle.me", "post"),
"#class" : postmill.PostmillPostExtractor,
- "#pattern" : "^https://fantasyanime\.com/anime/neo-tokyo-dub$",
+ "#pattern" : r"^https://fantasyanime\.com/anime/neo-tokyo-dub$",
"#count" : 1,
},
diff --git a/test/results/wikiart.py b/test/results/wikiart.py
index 47eb3ec7..9ab13103 100644
--- a/test/results/wikiart.py
+++ b/test/results/wikiart.py
@@ -12,7 +12,7 @@ __tests__ = (
"#url" : "https://www.wikiart.org/en/thomas-cole",
"#category": ("", "wikiart", "artist"),
"#class" : wikiart.WikiartArtistExtractor,
- "#pattern" : "https://uploads\d+\.wikiart\.org/(\d+/)?images/thomas-cole/[\w()-]+\.(jpg|png)",
+ "#pattern" : r"https://uploads\d+\.wikiart\.org/(\d+/)?images/thomas-cole/[\w()-]+\.(jpg|png)",
"#count" : "> 100",
"albums" : None,
From 63f649cd92a1aa2e70df8a2edbdb180ccae49f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Mon, 1 Jan 2024 17:38:32 +0100
Subject: [PATCH 19/77] [idolcomplex] fix extraction & update URL patterns
(#5002)
---
gallery_dl/extractor/idolcomplex.py | 17 ++++++++------
test/results/idolcomplex.py | 36 ++++++++++++++++++++++++++---
2 files changed, 43 insertions(+), 10 deletions(-)
diff --git a/gallery_dl/extractor/idolcomplex.py b/gallery_dl/extractor/idolcomplex.py
index b9e2c3dd..f70a948c 100644
--- a/gallery_dl/extractor/idolcomplex.py
+++ b/gallery_dl/extractor/idolcomplex.py
@@ -34,8 +34,11 @@ class IdolcomplexExtractor(SankakuExtractor):
self.start_post = 0
def _init(self):
+ self.find_pids = re.compile(
+ r" href=[\"#]/\w\w/posts/([0-9a-f]+)"
+ ).findall
self.find_tags = re.compile(
- r'tag-type-([^"]+)">\s*
', '
')
- for url in text.extract_iter(
- main_post,
- ']+>\s*\s*]*?href="/[^?]*\?tags=([^"]+)'
).findall
def items(self):
@@ -149,8 +152,8 @@ class IdolcomplexTagExtractor(IdolcomplexExtractor):
subcategory = "tag"
directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}"
- pattern = r"(?:https?://)?idol\.sankakucomplex\.com/\?([^#]*)"
- example = "https://idol.sankakucomplex.com/?tags=TAGS"
+ pattern = BASE_PATTERN + r"/(?:posts/?)?\?([^#]*)"
+ example = "https://idol.sankakucomplex.com/en/posts?tags=TAGS"
per_page = 20
def __init__(self, match):
@@ -196,7 +199,8 @@ class IdolcomplexTagExtractor(IdolcomplexExtractor):
page = self.request(self.root, params=params, retries=10).text
pos = ((page.find('id="more-popular-posts-link"') + 1) or
(page.find('
Date: Mon, 1 Jan 2024 22:05:21 +0100
Subject: [PATCH 20/77] [manganelo] fix extraction & recognize '.to' TLDs
(#5005)
---
gallery_dl/extractor/manganelo.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/gallery_dl/extractor/manganelo.py b/gallery_dl/extractor/manganelo.py
index 46019ad8..232b98d4 100644
--- a/gallery_dl/extractor/manganelo.py
+++ b/gallery_dl/extractor/manganelo.py
@@ -10,7 +10,11 @@ from .common import ChapterExtractor, MangaExtractor
from .. import text
import re
-BASE_PATTERN = r"(?:https?://)?((?:chap|read|www\.|m\.)?mangan(?:at|el)o\.com)"
+BASE_PATTERN = (
+ r"(?:https?://)?"
+ r"((?:chap|read|www\.|m\.)?mangan(?:at|el)o"
+ r"\.(?:to|com))"
+)
class ManganeloBase():
@@ -67,10 +71,11 @@ class ManganeloChapterExtractor(ManganeloBase, ChapterExtractor):
def images(self, page):
page = text.extr(
- page, 'class="container-chapter-reader', '\n
Date: Mon, 1 Jan 2024 22:58:42 +0100
Subject: [PATCH 21/77] [twitter] raise error for invalid 'strategy' values
(#4953)
---
gallery_dl/extractor/twitter.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
index fdcefddc..aa9ab9f6 100644
--- a/gallery_dl/extractor/twitter.py
+++ b/gallery_dl/extractor/twitter.py
@@ -552,9 +552,11 @@ class TwitterTimelineExtractor(TwitterExtractor):
return self.api.user_media
if strategy == "tweets":
return self.api.user_tweets
+ if strategy == "media":
+ return self.api.user_media
if strategy == "with_replies":
return self.api.user_tweets_and_replies
- return self.api.user_media
+ raise exception.StopExtraction("Invalid strategy '%s'", strategy)
class TwitterTweetsExtractor(TwitterExtractor):
From ee65f3de437b2d782d4d05765ecf93aa6ce19387 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Tue, 2 Jan 2024 15:03:04 +0100
Subject: [PATCH 22/77] [docs] add parent>child example (#4621)
---
docs/gallery-dl-example.conf | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf
index c3f80493..cda584e3 100644
--- a/docs/gallery-dl-example.conf
+++ b/docs/gallery-dl-example.conf
@@ -176,16 +176,15 @@
"imgur":
{
- "#": "use different directory and filename formats when coming from a reddit post",
- "directory":
- {
- "'_reddit' in locals()": []
- },
- "filename":
- {
- "'_reddit' in locals()": "{_reddit[id]} {id}.{extension}",
- "" : "{id}.{extension}"
- }
+ "#": "general imgur settings",
+ "filename": "{id}.{extension}"
+ },
+
+ "reddit>imgur":
+ {
+ "#": "special settings for imgur URLs found in reddit posts",
+ "directory": [],
+ "filename": "{_reddit[id]} {_reddit[title]} {id}.{extension}"
},
"tumblr":
From 4f3671458efc2d4f91baf31d2a1cfc54055872c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Tue, 2 Jan 2024 23:45:59 +0100
Subject: [PATCH 23/77] [deviantart] add 'avatar' and 'background' extractors
(#4995)
---
docs/configuration.rst | 8 ++-
docs/supportedsites.md | 2 +-
gallery_dl/extractor/deviantart.py | 56 ++++++++++++++--
test/results/deviantart.py | 100 ++++++++++++++++++++++++++++-
4 files changed, 154 insertions(+), 12 deletions(-)
diff --git a/docs/configuration.rst b/docs/configuration.rst
index e922f813..cbc54a7d 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -1401,7 +1401,13 @@ Description
when processing a user profile.
Possible values are
- ``"gallery"``, ``"scraps"``, ``"journal"``, ``"favorite"``, ``"status"``.
+ ``"avatar"``,
+ ``"background"``,
+ ``"gallery"``,
+ ``"scraps"``,
+ ``"journal"``,
+ ``"favorite"``,
+ ``"status"``.
It is possible to use ``"all"`` instead of listing all values separately.
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b538749b..dbdaac24 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -148,7 +148,7 @@ Consider all listed sites to potentially be NSFW.
DeviantArt
https://www.deviantart.com/
- Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches
+ Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches
OAuth
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index 2ba47e1e..4b5f1d77 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -38,7 +38,7 @@ class DeviantartExtractor(Extractor):
def __init__(self, match):
Extractor.__init__(self, match)
- self.user = match.group(1) or match.group(2)
+ self.user = (match.group(1) or match.group(2)).lower()
self.offset = 0
def _init(self):
@@ -104,7 +104,6 @@ class DeviantartExtractor(Extractor):
raise exception.StopExtraction()
else:
self.subcategory = "group-" + self.subcategory
- self.user = self.user.lower()
self.group = True
for deviation in self.deviations():
@@ -513,11 +512,13 @@ class DeviantartUserExtractor(DeviantartExtractor):
def items(self):
base = "{}/{}/".format(self.root, self.user)
return self._dispatch_extractors((
- (DeviantartGalleryExtractor , base + "gallery"),
- (DeviantartScrapsExtractor , base + "gallery/scraps"),
- (DeviantartJournalExtractor , base + "posts"),
- (DeviantartStatusExtractor , base + "posts/statuses"),
- (DeviantartFavoriteExtractor, base + "favourites"),
+ (DeviantartAvatarExtractor , base + "avatar"),
+ (DeviantartBackgroundExtractor, base + "banner"),
+ (DeviantartGalleryExtractor , base + "gallery"),
+ (DeviantartScrapsExtractor , base + "gallery/scraps"),
+ (DeviantartJournalExtractor , base + "posts"),
+ (DeviantartStatusExtractor , base + "posts/statuses"),
+ (DeviantartFavoriteExtractor , base + "favourites"),
), ("gallery",))
@@ -538,6 +539,47 @@ class DeviantartGalleryExtractor(DeviantartExtractor):
return self._folder_urls(folders, "gallery", DeviantartFolderExtractor)
+class DeviantartAvatarExtractor(DeviantartExtractor):
+ """Extractor for an artist's avatar"""
+ subcategory = "avatar"
+ archive_fmt = "a_{_username}_{index}"
+ pattern = BASE_PATTERN + r"/avatar"
+ example = "https://www.deviantart.com/USER/avatar/"
+
+ def deviations(self):
+ profile = self.api.user_profile(self.user.lower())
+ if profile:
+ url = profile["user"]["usericon"]
+ return ({
+ "author" : profile["user"],
+ "category" : "avatar",
+ "index" : text.parse_int(url.rpartition("?")[2]),
+ "is_deleted" : False,
+ "is_downloadable": False,
+ "published_time" : 0,
+ "title" : "avatar",
+ "content" : {
+ "src": url.replace("/avatars/", "/avatars-big/", 1),
+ },
+ },)
+ return ()
+
+
+class DeviantartBackgroundExtractor(DeviantartExtractor):
+ """Extractor for an artist's banner"""
+ subcategory = "background"
+ archive_fmt = "b_{index}"
+ pattern = BASE_PATTERN + r"/ba(?:nner|ckground)"
+ example = "https://www.deviantart.com/USER/banner/"
+
+ def deviations(self):
+ try:
+ return (self.api.user_profile(self.user.lower())
+ ["cover_deviation"]["cover_deviation"],)
+ except Exception:
+ return ()
+
+
class DeviantartFolderExtractor(DeviantartExtractor):
"""Extractor for deviations inside an artist's gallery folder"""
subcategory = "folder"
diff --git a/test/results/deviantart.py b/test/results/deviantart.py
index 4196f32c..45ee6c18 100644
--- a/test/results/deviantart.py
+++ b/test/results/deviantart.py
@@ -14,7 +14,7 @@ __tests__ = (
"#url" : "https://www.deviantart.com/shimoda7",
"#category": ("", "deviantart", "user"),
"#class" : deviantart.DeviantartUserExtractor,
- "#pattern" : "/shimoda7/gallery$",
+ "#urls" : "https://www.deviantart.com/shimoda7/gallery",
},
{
@@ -22,8 +22,15 @@ __tests__ = (
"#category": ("", "deviantart", "user"),
"#class" : deviantart.DeviantartUserExtractor,
"#options" : {"include": "all"},
- "#pattern" : "/shimoda7/(gallery(/scraps)?|posts(/statuses)?|favourites)$",
- "#count" : 5,
+ "#urls" : (
+ "https://www.deviantart.com/shimoda7/avatar",
+ "https://www.deviantart.com/shimoda7/banner",
+ "https://www.deviantart.com/shimoda7/gallery",
+ "https://www.deviantart.com/shimoda7/gallery/scraps",
+ "https://www.deviantart.com/shimoda7/posts",
+ "https://www.deviantart.com/shimoda7/posts/statuses",
+ "https://www.deviantart.com/shimoda7/favourites",
+ ),
},
{
@@ -195,6 +202,93 @@ __tests__ = (
"#class" : deviantart.DeviantartGalleryExtractor,
},
+{
+ "#url" : "https://deviantart.com/shimoda7/avatar",
+ "#category": ("", "deviantart", "avatar"),
+ "#class" : deviantart.DeviantartAvatarExtractor,
+ "#urls" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4",
+ "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f",
+
+ "author" : {
+ "type" : "premium",
+ "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4",
+ "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
+ "username": "shimoda7",
+ },
+ "content" : {
+ "src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"
+ },
+ "da_category" : "avatar",
+ "date" : "dt:1970-01-01 00:00:00",
+ "extension" : "jpg",
+ "filename" : "avatar_by_shimoda7-d4",
+ "index" : 4,
+ "index_base36" : "4",
+ "is_deleted" : False,
+ "is_downloadable": False,
+ "is_original" : True,
+ "published_time" : 0,
+ "target" : {
+ "extension": "jpg",
+ "filename" : "avatar_by_shimoda7-d4",
+ "src" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"
+ },
+ "title" : "avatar",
+ "username" : "shimoda7",
+},
+
+{
+ "#url" : "https://deviantart.com/gdldev/banner",
+ "#category": ("", "deviantart", "background"),
+ "#class" : deviantart.DeviantartBackgroundExtractor,
+ "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$",
+ "#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f",
+
+ "allows_comments" : True,
+ "author" : {
+ "type" : "regular",
+ "usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?2",
+ "userid" : "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E",
+ "username": "gdldev"
+ },
+ "category_path" : "",
+ "content" : {
+ "filename" : "banner_by_gdldev_dgntyqc.png",
+ "filesize" : 84510,
+ "height" : 4000,
+ "src" : r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$",
+ "transparency": False,
+ "width" : 6400
+ },
+ "da_category" : "Uncategorized",
+ "date" : "dt:2024-01-02 21:16:06",
+ "deviationid" : "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A",
+ "download_filesize": 84510,
+ "extension" : "png",
+ "filename" : "banner_by_gdldev-dgntyqc",
+ "index" : 1007488020,
+ "index_base36" : "gntyqc",
+ "is_blocked" : False,
+ "is_deleted" : False,
+ "is_downloadable" : True,
+ "is_favourited" : False,
+ "is_mature" : False,
+ "is_original" : True,
+ "is_published" : False,
+ "preview" : dict,
+ "printid" : None,
+ "published_time" : 1704230166,
+ "stats" : {
+ "comments" : 0,
+ "favourites": 0,
+ },
+ "target" : dict,
+ "thumbs" : list,
+ "title" : "Banner",
+ "url" : "https://sta.sh/0198jippkeys",
+ "username" : "gdldev",
+},
+
{
"#url" : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous",
"#comment" : "user",
From 00570028a365b514b8636d434112ad30e333f9a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Wed, 3 Jan 2024 01:25:50 +0100
Subject: [PATCH 24/77] [cookies] fix macOS Firefox profile path
https://github.com/yt-dlp/yt-dlp/commit/85b33f5c163f60dbd089a6b9bc2ba1366d3ddf93
---
gallery_dl/cookies.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py
index 416cc9a1..478abb63 100644
--- a/gallery_dl/cookies.py
+++ b/gallery_dl/cookies.py
@@ -215,9 +215,11 @@ def _firefox_cookies_database(profile=None, container=None):
def _firefox_browser_directory():
if sys.platform in ("win32", "cygwin"):
- return os.path.expandvars(r"%APPDATA%\Mozilla\Firefox\Profiles")
+ return os.path.expandvars(
+ r"%APPDATA%\Mozilla\Firefox\Profiles")
if sys.platform == "darwin":
- return os.path.expanduser("~/Library/Application Support/Firefox")
+ return os.path.expanduser(
+ "~/Library/Application Support/Firefox/Profiles")
return os.path.expanduser("~/.mozilla/firefox")
From 7eaf648f2e937114d5c3a2c60a98b3469acb5b40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Thu, 4 Jan 2024 15:01:33 +0100
Subject: [PATCH 25/77] [fanbox] add 'metadata' option (#4921)
extracts 'plan' and extended 'user' metadata
---
docs/configuration.rst | 17 +++++++++-
gallery_dl/extractor/fanbox.py | 59 +++++++++++++++++++++++++++++++---
test/results/fanbox.py | 37 +++++++++++++++++++++
3 files changed, 107 insertions(+), 6 deletions(-)
diff --git a/docs/configuration.rst b/docs/configuration.rst
index cbc54a7d..8a1752ee 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -1569,7 +1569,7 @@ Default
``false``
Example
* ``notes,pools``
- * ``["notes", "pools"``
+ * ``["notes", "pools"]``
Description
Extract additional metadata (notes, pool metadata) if available.
@@ -1711,6 +1711,21 @@ Description
* ``false``: Ignore embeds.
+extractor.fanbox.metadata
+-------------------------
+Type
+ * ``bool``
+ * ``string``
+ * ``list`` of ``strings``
+Default
+ ``false``
+Example
+ * ``user,plan``
+ * ``["user", "plan"]``
+Description
+ Extract ``plan`` and extended ``user`` metadata.
+
+
extractor.flickr.access-token & .access-token-secret
----------------------------------------------------
Type
diff --git a/gallery_dl/extractor/fanbox.py b/gallery_dl/extractor/fanbox.py
index 4572bea6..61a39283 100644
--- a/gallery_dl/extractor/fanbox.py
+++ b/gallery_dl/extractor/fanbox.py
@@ -8,6 +8,7 @@
from .common import Extractor, Message
from .. import text
+from ..cache import memcache
import re
BASE_PATTERN = (
@@ -27,8 +28,20 @@ class FanboxExtractor(Extractor):
_warning = True
def _init(self):
+ self.headers = {"Origin": self.root}
self.embeds = self.config("embeds", True)
+ includes = self.config("metadata")
+ if includes:
+ if isinstance(includes, str):
+ includes = includes.split(",")
+ elif not isinstance(includes, (list, tuple)):
+ includes = ("user", "plan")
+ self._meta_user = ("user" in includes)
+ self._meta_plan = ("plan" in includes)
+ else:
+ self._meta_user = self._meta_plan = False
+
if self._warning:
if not self.cookies_check(("FANBOXSESSID",)):
self.log.warning("no 'FANBOXSESSID' cookie set")
@@ -43,11 +56,9 @@ class FanboxExtractor(Extractor):
"""Return all relevant post objects"""
def _pagination(self, url):
- headers = {"Origin": self.root}
-
while url:
url = text.ensure_http_scheme(url)
- body = self.request(url, headers=headers).json()["body"]
+ body = self.request(url, headers=self.headers).json()["body"]
for item in body["items"]:
try:
yield self._get_post_data(item["id"])
@@ -58,9 +69,8 @@ class FanboxExtractor(Extractor):
def _get_post_data(self, post_id):
"""Fetch and process post data"""
- headers = {"Origin": self.root}
url = "https://api.fanbox.cc/post.info?postId="+post_id
- post = self.request(url, headers=headers).json()["body"]
+ post = self.request(url, headers=self.headers).json()["body"]
content_body = post.pop("body", None)
if content_body:
@@ -98,8 +108,47 @@ class FanboxExtractor(Extractor):
post["text"] = content_body.get("text") if content_body else None
post["isCoverImage"] = False
+ if self._meta_user:
+ post["user"] = self._get_user_data(post["creatorId"])
+ if self._meta_plan:
+ plans = self._get_plan_data(post["creatorId"])
+ post["plan"] = plans[post["feeRequired"]]
+
return content_body, post
+ @memcache(keyarg=1)
+ def _get_user_data(self, creator_id):
+ url = "https://api.fanbox.cc/creator.get"
+ params = {"creatorId": creator_id}
+ data = self.request(url, params=params, headers=self.headers).json()
+
+ user = data["body"]
+ user.update(user.pop("user"))
+
+ return user
+
+ @memcache(keyarg=1)
+ def _get_plan_data(self, creator_id):
+ url = "https://api.fanbox.cc/plan.listCreator"
+ params = {"creatorId": creator_id}
+ data = self.request(url, params=params, headers=self.headers).json()
+
+ plans = {0: {
+ "id" : "",
+ "title" : "",
+ "fee" : 0,
+ "description" : "",
+ "coverImageUrl" : "",
+ "creatorId" : creator_id,
+ "hasAdultContent": None,
+ "paymentMethod" : None,
+ }}
+ for plan in data["body"]:
+ del plan["user"]
+ plans[plan["fee"]] = plan
+
+ return plans
+
def _get_urls_from_post(self, content_body, post):
num = 0
cover_image = post.get("coverImageUrl")
diff --git a/test/results/fanbox.py b/test/results/fanbox.py
index 78f7fe54..32f13096 100644
--- a/test/results/fanbox.py
+++ b/test/results/fanbox.py
@@ -86,6 +86,43 @@ __tests__ = (
"content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, September 5th, 2022, we are happy to announce the start of the FANBOX hashtag event #MySetupTour ! \nAbout the event\nTo join this event .+ \nPlease check this page for further details regarding the Privacy & Terms.\nhttps://fanbox.pixiv.help/.+/10184952456601\n\n\nThank you for your continued support of FANBOX.$",
},
+{
+ "#url" : "https://official-en.fanbox.cc/posts/7022572",
+ "#comment" : "'plan' and 'user' metadata (#4921)",
+ "#category": ("", "fanbox", "post"),
+ "#class" : fanbox.FanboxPostExtractor,
+ "#options" : {"metadata": True},
+
+ "plan": {
+ "coverImageUrl" : "",
+ "creatorId" : "official-en",
+ "description" : "",
+ "fee" : 0,
+ "hasAdultContent": None,
+ "id" : "",
+ "paymentMethod" : None,
+ "title" : "",
+ },
+ "user": {
+ "coverImageUrl" : "https://pixiv.pximg.net/c/1620x580_90_a2_g5/fanbox/public/images/creator/74349833/cover/n9mX8q4tUXHXXj7sK1RPWyUu.jpeg",
+ "creatorId" : "official-en",
+ "description" : "This is the official English pixivFANBOX account! \n(official Japanese account: https://official.fanbox.cc/ )\n\npixivFANBOX is a subscription service for building a reliable fan community where creators can nurture creative lifestyles together with their fans.\nFollowers can be notified of the updates from their favorite creators they are following. Supporters can enjoy closer communication with creators through exclusive content and their latest information.\n",
+ "hasAdultContent" : False,
+ "hasBoothShop" : False,
+ "iconUrl" : "https://pixiv.pximg.net/c/160x160_90_a2_g5/fanbox/public/images/user/74349833/icon/oJH0OoGoSixLrJXlnneNvC95.jpeg",
+ "isAcceptingRequest": False,
+ "isFollowed" : False,
+ "isStopped" : False,
+ "isSupported" : False,
+ "name" : "pixivFANBOX English",
+ "profileItems" : [],
+ "profileLinks" : [
+ "https://twitter.com/pixivfanbox",
+ ],
+ "userId" : "74349833",
+ },
+},
+
{
"#url" : "https://mochirong.fanbox.cc/posts/3746116",
"#comment" : "imageMap file order (#2718)",
From a86775f6175460ea6ceb963567000c2b6e7002fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Thu, 4 Jan 2024 15:05:33 +0100
Subject: [PATCH 26/77] [gelbooru] fix 'favorite' extractor (#4903)
lots of +1/-1 and = last:
+ pnum, last = divmod(count-1, self.per_page)
+ if self.offset > last:
+ # page number change
self.offset -= last
- diff, self.offset = divmod(self.offset, self.per_page)
+ diff, self.offset = divmod(self.offset-1, self.per_page)
pnum -= diff + 1
skip = self.offset
@@ -183,8 +184,8 @@ class GelbooruFavoriteExtractor(GelbooruBase,
while True:
favs = self._api_request(params, "favorite")
-
favs.reverse()
+
if skip:
favs = favs[skip:]
skip = 0
From 0f3013610997d67c6c952e2974597af7797e6957 Mon Sep 17 00:00:00 2001
From: enduser420 <91022934+enduser420@users.noreply.github.com>
Date: Thu, 4 Jan 2024 21:38:59 +0530
Subject: [PATCH 27/77] [zzup] add 'gallery' extractor
---
gallery_dl/extractor/__init__.py | 1 +
gallery_dl/extractor/zzup.py | 40 ++++++++++++++++++++++++++++++++
test/results/zzup.py | 31 +++++++++++++++++++++++++
3 files changed, 72 insertions(+)
create mode 100644 gallery_dl/extractor/zzup.py
create mode 100644 test/results/zzup.py
diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py
index 9c684bc0..8d974ecc 100644
--- a/gallery_dl/extractor/__init__.py
+++ b/gallery_dl/extractor/__init__.py
@@ -178,6 +178,7 @@ modules = [
"xhamster",
"xvideos",
"zerochan",
+ "zzup",
"booru",
"moebooru",
"foolfuuka",
diff --git a/gallery_dl/extractor/zzup.py b/gallery_dl/extractor/zzup.py
new file mode 100644
index 00000000..45b0cd80
--- /dev/null
+++ b/gallery_dl/extractor/zzup.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from .common import GalleryExtractor
+from .. import text
+
+
+class ZzupGalleryExtractor(GalleryExtractor):
+ category = "zzup"
+ directory_fmt = ("{category}", "{title}")
+ filename_fmt = "{slug}_{num:>03}.{extension}"
+ archive_fmt = "{slug}_{num}"
+ root = "https://zzup.com"
+ pattern = (r"(?:https?://)?(?:www\.)?zzup\.com(/content"
+ r"/[\w=]+/([^/?#]+)/[\w=]+)/(?:index|page-\d+)\.html")
+ example = "https://zzup.com/content/xyz=/12345_TITLE/123=/index.html"
+
+ def __init__(self, match):
+ url = "{}/{}/index.html".format(self.root, match.group(1))
+ GalleryExtractor.__init__(self, match, url)
+ self.slug = match.group(2)
+
+ def metadata(self, page):
+ return {
+ "slug" : self.slug,
+ "title": text.unescape(text.extr(
+ page, "", " "))[:-11],
+ }
+
+ def images(self, page):
+ path = text.extr(page, 'class="picbox">05}" + p2[4:]
+ return [(ufmt.format(num), None) for num in range(1, count + 1)]
diff --git a/test/results/zzup.py b/test/results/zzup.py
new file mode 100644
index 00000000..ad68e41c
--- /dev/null
+++ b/test/results/zzup.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import zzup
+
+
+__tests__ = (
+{
+ "#url" : "https://zzup.com/content/NjM=/MetArt_20080206_viki_c_sensazioni_by_ingret/OTE=/index.html",
+ "#category": ("", "zzup", "gallery"),
+ "#class" : zzup.ZzupGalleryExtractor,
+ "#pattern" : r"https://zzup\.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image00\d\d\d-5896498214-1-9689595623/MetArt-20080206_viki_c_sensazioni_by_ingret/9879560327/zzup.com.jpg",
+
+ "slug" : "MetArt_20080206_viki_c_sensazioni_by_ingret",
+ "title" : "MetArt 20080206 viki c sensazioni by ingret",
+ "num" : int,
+ "count" : 135,
+},
+
+{
+ "#url" : "https://zzup.com/content/MTc2MDYxMw==/Courtesan/NDA=/page-1.html",
+ "#category": ("", "zzup", "gallery"),
+ "#class" : zzup.ZzupGalleryExtractor,
+ "#pattern" : r"https://zzup.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image000\d\d-5896498214-40-9689595623/Courtesan/9879560327/zzup.com.jpg",
+},
+
+)
+
From 0ab0a10d2dff8a32177f45c23f48c58e5493b725 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Fri, 5 Jan 2024 02:26:22 +0100
Subject: [PATCH 28/77] [jpgfish] update domain
---
gallery_dl/extractor/chevereto.py | 2 +-
test/results/jpgfish.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py
index 21166bdb..2bf200b0 100644
--- a/gallery_dl/extractor/chevereto.py
+++ b/gallery_dl/extractor/chevereto.py
@@ -35,7 +35,7 @@ class CheveretoExtractor(BaseExtractor):
BASE_PATTERN = CheveretoExtractor.update({
"jpgfish": {
- "root": "https://jpg2.su",
+ "root": "https://jpg4.su",
"pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)",
},
"pixl": {
diff --git a/test/results/jpgfish.py b/test/results/jpgfish.py
index bf35bf7a..354e2ff5 100644
--- a/test/results/jpgfish.py
+++ b/test/results/jpgfish.py
@@ -9,7 +9,7 @@ from gallery_dl.extractor import chevereto
__tests__ = (
{
- "#url" : "https://jpg2.su/img/funnymeme.LecXGS",
+ "#url" : "https://jpg4.su/img/funnymeme.LecXGS",
"#category": ("chevereto", "jpgfish", "image"),
"#class" : chevereto.CheveretoImageExtractor,
"#urls" : "https://simp3.jpg.church/images/funnymeme.jpg",
From b4bcf40278e79628a81f58d7640f41630a9c66b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Fri, 5 Jan 2024 17:18:33 +0100
Subject: [PATCH 29/77] [weibo] fix AttributeError in 'user' extractor (#5022)
yet another bug caused by a383eca7
---
gallery_dl/extractor/weibo.py | 3 ---
test/results/weibo.py | 47 +++++++++++++++++++++++++++++------
2 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py
index 7413b5a0..3bd06489 100644
--- a/gallery_dl/extractor/weibo.py
+++ b/gallery_dl/extractor/weibo.py
@@ -225,9 +225,6 @@ class WeiboUserExtractor(WeiboExtractor):
pattern = USER_PATTERN + r"(?:$|#)"
example = "https://weibo.com/USER"
- def initialize(self):
- pass
-
def items(self):
base = "{}/u/{}?tabtype=".format(self.root, self._user_id())
return self._dispatch_extractors((
diff --git a/test/results/weibo.py b/test/results/weibo.py
index 639994c0..68e27f8f 100644
--- a/test/results/weibo.py
+++ b/test/results/weibo.py
@@ -13,7 +13,35 @@ __tests__ = (
"#url" : "https://weibo.com/1758989602",
"#category": ("", "weibo", "user"),
"#class" : weibo.WeiboUserExtractor,
- "#pattern" : r"^https://weibo\.com/u/1758989602\?tabtype=feed$",
+ "#urls" : "https://weibo.com/u/1758989602?tabtype=feed",
+},
+
+{
+ "#url" : "https://weibo.com/1758989602",
+ "#category": ("", "weibo", "user"),
+ "#class" : weibo.WeiboUserExtractor,
+ "#options" : {"include": "all"},
+ "#urls" : (
+ "https://weibo.com/u/1758989602?tabtype=home",
+ "https://weibo.com/u/1758989602?tabtype=feed",
+ "https://weibo.com/u/1758989602?tabtype=video",
+ "https://weibo.com/u/1758989602?tabtype=newVideo",
+ "https://weibo.com/u/1758989602?tabtype=album",
+ ),
+},
+
+{
+ "#url" : "https://weibo.com/zhouyuxi77",
+ "#category": ("", "weibo", "user"),
+ "#class" : weibo.WeiboUserExtractor,
+ "#urls" : "https://weibo.com/u/7488709788?tabtype=feed",
+},
+
+{
+ "#url" : "https://www.weibo.com/n/周于希Sally",
+ "#category": ("", "weibo", "user"),
+ "#class" : weibo.WeiboUserExtractor,
+ "#urls" : "https://weibo.com/u/7488709788?tabtype=feed",
},
{
@@ -69,9 +97,11 @@ __tests__ = (
"#class" : weibo.WeiboFeedExtractor,
"#range" : "1",
- "status": {"user": {
- "id" : 7488709788,
-}},
+ "status": {
+ "user": {
+ "id": 7488709788,
+ },
+ },
},
{
@@ -80,9 +110,12 @@ __tests__ = (
"#class" : weibo.WeiboFeedExtractor,
"#range" : "1",
- "status": {"user": {
- "id" : 7488709788,
-}},
+
+ "status": {
+ "user": {
+ "id": 7488709788,
+ },
+ },
},
{
From e61f016465c0edd71725c11dadfb66da57decce5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?=
Date: Fri, 5 Jan 2024 17:56:39 +0100
Subject: [PATCH 30/77] [szurubooru] support 'snootbooru.com' (#5023)
---
docs/supportedsites.md | 8 ++-
gallery_dl/extractor/szurubooru.py | 4 ++
test/results/snootbooru.py | 79 ++++++++++++++++++++++++++++++
3 files changed, 90 insertions(+), 1 deletion(-)
create mode 100644 test/results/snootbooru.py
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dbdaac24..c0fee2a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1037,7 +1037,7 @@ Consider all listed sites to potentially be NSFW.
JPG Fish
- https://jpg2.su/
+ https://jpg4.su/
Albums, individual Images, User Profiles
@@ -1409,6 +1409,12 @@ Consider all listed sites to potentially be NSFW.
Posts, Tag Searches