Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2025-01-31 19:51:34 +01:00
Code Issues Releases Wiki Activity

remove test results in extractor modules

Browse Source 
and add generic example URLs
This commit is contained in:
Mike Fährmann 2023-09-11 16:30:55 +02:00
parent a833c244c8
commit a453335a9f
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
176 changed files with 656 additions and 9554 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
gallery_dl/extractor/2chan.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2017-2022 Mike Fährmann # Copyright 2017-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -20,26 +20,8 @@ class _2chanThreadExtractor(Extractor):
filename_fmt = "{tim}.{extension}" filename_fmt = "{tim}.{extension}"
archive_fmt = "{board}_{thread}_{tim}" archive_fmt = "{board}_{thread}_{tim}"
url_fmt = "https://{server}.2chan.net/{board}/src/{filename}" url_fmt = "https://{server}.2chan.net/{board}/src/{filename}"
pattern = r"(?:https?://)?([\w-]+)\.2chan\.net/([^/]+)/res/(\d+)" pattern = r"(?:https?://)?([\w-]+)\.2chan\.net/([^/?#]+)/res/(\d+)"
test = ("https://dec.2chan.net/70/res/14565.htm", { example = "https://dec.2chan.net/12/res/12345.htm"
"pattern": r"https://dec\.2chan\.net/70/src/\d{13}\.jpg",
"count": ">= 3",
"keyword": {
"board": "70",
"board_name": "新板提案",
"com": str,
"fsize": r"re:\d+",
"name": "名無し",
"no": r"re:1[45]\d\d\d",
"now": r"re:22/../..\(.\)..:..:..",
"post": "無題",
"server": "dec",
"thread": "14565",
"tim": r"re:^\d{13}$",
"time": r"re:^\d{10}$",
"title": "ﾋﾛｱｶ板"
},
})
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

30
gallery_dl/extractor/2chen.py
View File

@ -21,26 +21,7 @@ class _2chenThreadExtractor(Extractor):
filename_fmt = "{time} {filename}.{extension}" filename_fmt = "{time} {filename}.{extension}"
archive_fmt = "{board}_{thread}_{hash}_{time}" archive_fmt = "{board}_{thread}_{hash}_{time}"
pattern = BASE_PATTERN + r"/([^/?#]+)/(\d+)" pattern = BASE_PATTERN + r"/([^/?#]+)/(\d+)"
test = ( example = "https://sturdychan.help/a/12345/"
("https://sturdychan.help/tv/268929", {
"pattern": r"https://sturdychan\.help/assets/images"
r"/src/\w{40}\.\w+$",
"count": ">= 179",
"keyword": {
"board": "tv",
"date": "type:datetime",
"hash": r"re:[0-9a-f]{40}",
"name": "Anonymous",
"no": r"re:\d+",
"thread": "268929",
"time": int,
"title": "「/ttg/ #118: 🇧🇷 edition」",
"url": str,
},
}),
("https://2chen.club/tv/1"),
("https://2chen.moe/jp/303786"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
@ -101,14 +82,7 @@ class _2chenBoardExtractor(Extractor):
subcategory = "board" subcategory = "board"
root = "https://sturdychan.help" root = "https://sturdychan.help"
pattern = BASE_PATTERN + r"/([^/?#]+)(?:/catalog|/?$)" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/catalog|/?$)"
test = ( example = "https://sturdychan.help/a/"
("https://sturdychan.help/co/", {
"pattern": _2chenThreadExtractor.pattern
}),
("https://2chen.moe/co"),
("https://2chen.club/tv"),
("https://2chen.moe/co/catalog"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

44
gallery_dl/extractor/35photo.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2019-2022 Mike Fährmann # Copyright 2019-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -101,20 +101,7 @@ class _35photoUserExtractor(_35photoExtractor):
subcategory = "user" subcategory = "user"
pattern = (r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro" pattern = (r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro"
r"/(?!photo_|genre_|tags/|rating/)([^/?#]+)") r"/(?!photo_|genre_|tags/|rating/)([^/?#]+)")
test = ( example = "https://35photo.pro/USER"
("https://35photo.pro/liya", {
"pattern": r"https://([a-z][0-9]\.)?35photo\.pro"
r"/photos_(main|series)/.*\.jpg",
"count": 9,
}),
("https://35photo.pro/suhoveev", {
# last photo ID (1267028) isn't given as 'photo-id="<id>"
# there are only 23 photos without the last one
"count": ">= 33",
}),
("https://en.35photo.pro/liya"),
("https://ru.35photo.pro/liya"),
)
def __init__(self, match): def __init__(self, match):
_35photoExtractor.__init__(self, match) _35photoExtractor.__init__(self, match)
@ -143,11 +130,7 @@ class _35photoTagExtractor(_35photoExtractor):
directory_fmt = ("{category}", "Tags", "{search_tag}") directory_fmt = ("{category}", "Tags", "{search_tag}")
archive_fmt = "t{search_tag}_{id}_{num}" archive_fmt = "t{search_tag}_{id}_{num}"
pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/tags/([^/?#]+)" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/tags/([^/?#]+)"
test = ("https://35photo.pro/tags/landscape/", { example = "https://35photo.pro/tags/TAG/"
"range": "1-25",
"count": 25,
"archive": False,
})
def __init__(self, match): def __init__(self, match):
_35photoExtractor.__init__(self, match) _35photoExtractor.__init__(self, match)
@ -180,7 +163,7 @@ class _35photoGenreExtractor(_35photoExtractor):
directory_fmt = ("{category}", "Genre", "{genre}") directory_fmt = ("{category}", "Genre", "{genre}")
archive_fmt = "g{genre_id}_{id}_{num}" archive_fmt = "g{genre_id}_{id}_{num}"
pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/genre_(\d+)(/new/)?" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/genre_(\d+)(/new/)?"
test = ("https://35photo.pro/genre_109/",) example = "https://35photo.pro/genre_12345/"
def __init__(self, match): def __init__(self, match):
_35photoExtractor.__init__(self, match) _35photoExtractor.__init__(self, match)
@ -212,24 +195,7 @@ class _35photoImageExtractor(_35photoExtractor):
"""Extractor for individual images from 35photo.pro""" """Extractor for individual images from 35photo.pro"""
subcategory = "image" subcategory = "image"
pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/photo_(\d+)" pattern = r"(?:https?://)?(?:[a-z]+\.)?35photo\.pro/photo_(\d+)"
test = ("https://35photo.pro/photo_753340/", { example = "https://35photo.pro/photo_12345/"
"count": 1,
"keyword": {
"url" : r"re:https://35photo\.pro/photos_main/.*\.jpg",
"id" : 753340,
"title" : "Winter walk",
"description": str,
"tags" : list,
"views" : int,
"favorites" : int,
"score" : int,
"type" : 0,
"date" : "15 авг, 2014",
"user" : "liya",
"user_id" : 20415,
"user_name" : "Liya Mirzaeva",
},
})
def __init__(self, match): def __init__(self, match):
_35photoExtractor.__init__(self, match) _35photoExtractor.__init__(self, match)

27
gallery_dl/extractor/3dbooru.py
View File

@ -27,10 +27,7 @@ class _3dbooruTagExtractor(_3dbooruBase, moebooru.MoebooruTagExtractor):
"""Extractor for images from behoimi.org based on search-tags""" """Extractor for images from behoimi.org based on search-tags"""
pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org/post" pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org/post"
r"(?:/(?:index)?)?\?tags=(?P<tags>[^&#]+)") r"(?:/(?:index)?)?\?tags=(?P<tags>[^&#]+)")
test = ("http://behoimi.org/post?tags=himekawa_azuru+dress", { example = "http://behoimi.org/post?tags=TAG"
"url": "ecb30c6aaaf8a6ff8f55255737a9840832a483c1",
"content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a",
})
def posts(self): def posts(self):
params = {"tags": self.tags} params = {"tags": self.tags}
@ -40,10 +37,7 @@ class _3dbooruTagExtractor(_3dbooruBase, moebooru.MoebooruTagExtractor):
class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor): class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor):
"""Extractor for image-pools from behoimi.org""" """Extractor for image-pools from behoimi.org"""
pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(?P<pool>\d+)" pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(?P<pool>\d+)"
test = ("http://behoimi.org/pool/show/27", { example = "http://behoimi.org/pool/show/12345"
"url": "da75d2d1475449d5ef0c266cb612683b110a30f2",
"content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554",
})
def posts(self): def posts(self):
params = {"tags": "pool:" + self.pool_id} params = {"tags": "pool:" + self.pool_id}
@ -53,17 +47,7 @@ class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor):
class _3dbooruPostExtractor(_3dbooruBase, moebooru.MoebooruPostExtractor): class _3dbooruPostExtractor(_3dbooruBase, moebooru.MoebooruPostExtractor):
"""Extractor for single images from behoimi.org""" """Extractor for single images from behoimi.org"""
pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(?P<post>\d+)" pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(?P<post>\d+)"
test = ("http://behoimi.org/post/show/140852", { example = "http://behoimi.org/post/show/12345"
"url": "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6",
"content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4",
"options": (("tags", True),),
"keyword": {
"tags_character": "furude_rika",
"tags_copyright": "higurashi_no_naku_koro_ni",
"tags_model": "himekawa_azuru",
"tags_general": str,
},
})
def posts(self): def posts(self):
params = {"tags": "id:" + self.post_id} params = {"tags": "id:" + self.post_id}
@ -76,7 +60,4 @@ class _3dbooruPopularExtractor(
pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org" pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org"
r"/post/popular_(?P<scale>by_(?:day|week|month)|recent)" r"/post/popular_(?P<scale>by_(?:day|week|month)|recent)"
r"(?:\?(?P<query>[^#]*))?") r"(?:\?(?P<query>[^#]*))?")
test = ("http://behoimi.org/post/popular_by_month?month=2&year=2013", { example = "http://behoimi.org/post/popular_by_month"
"pattern": r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg",
"count": 20,
})

19
gallery_dl/extractor/4chan.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2015-2019 Mike Fährmann # Copyright 2015-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -21,17 +21,7 @@ class _4chanThreadExtractor(Extractor):
archive_fmt = "{board}_{thread}_{tim}" archive_fmt = "{board}_{thread}_{tim}"
pattern = (r"(?:https?://)?boards\.4chan(?:nel)?\.org" pattern = (r"(?:https?://)?boards\.4chan(?:nel)?\.org"
r"/([^/]+)/thread/(\d+)") r"/([^/]+)/thread/(\d+)")
test = ( example = "https://boards.4channel.org/a/thread/12345/"
("https://boards.4chan.org/tg/thread/15396072/", {
"url": "39082ad166161966d7ba8e37f2173a824eb540f0",
"keyword": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a",
"content": "20b7b51afa51c9c31a0020a0737b889532c8d7ec",
}),
("https://boards.4channel.org/tg/thread/15396072/", {
"url": "39082ad166161966d7ba8e37f2173a824eb540f0",
"keyword": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
@ -65,10 +55,7 @@ class _4chanBoardExtractor(Extractor):
category = "4chan" category = "4chan"
subcategory = "board" subcategory = "board"
pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org/([^/?#]+)/\d*$" pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org/([^/?#]+)/\d*$"
test = ("https://boards.4channel.org/po/", { example = "https://boards.4channel.org/a/"
"pattern": _4chanThreadExtractor.pattern,
"count": ">= 100",
})
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

26
gallery_dl/extractor/4chanarchives.py
View File

@ -21,21 +21,7 @@ class _4chanarchivesThreadExtractor(Extractor):
filename_fmt = "{no}-{filename}.{extension}" filename_fmt = "{no}-{filename}.{extension}"
archive_fmt = "{board}_{thread}_{no}" archive_fmt = "{board}_{thread}_{no}"
pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)/thread/(\d+)" pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)/thread/(\d+)"
test = ( example = "https://4chanarchives.com/board/a/thread/12345/"
("https://4chanarchives.com/board/c/thread/2707110", {
"pattern": r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg",
"count": 2,
"keyword": {
"board": "c",
"com": str,
"name": "Anonymous",
"no": int,
"thread": "2707110",
"time": r"re:2016-07-1\d \d\d:\d\d:\d\d",
"title": "Ren Kagami from 'Oyako Neburi'",
},
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
@ -106,15 +92,7 @@ class _4chanarchivesBoardExtractor(Extractor):
subcategory = "board" subcategory = "board"
root = "https://4chanarchives.com" root = "https://4chanarchives.com"
pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)(?:/(\d+))?/?$" pattern = r"(?:https?://)?4chanarchives\.com/board/([^/?#]+)(?:/(\d+))?/?$"
test = ( example = "https://4chanarchives.com/board/a/"
("https://4chanarchives.com/board/c/", {
"pattern": _4chanarchivesThreadExtractor.pattern,
"range": "1-40",
"count": 40,
}),
("https://4chanarchives.com/board/c"),
("https://4chanarchives.com/board/c/10"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

69
gallery_dl/extractor/500px.py
View File

@ -96,15 +96,7 @@ class _500pxUserExtractor(_500pxExtractor):
"""Extractor for photos from a user's photostream on 500px.com""" """Extractor for photos from a user's photostream on 500px.com"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/(?!photo/|liked)(?:p/)?([^/?#]+)/?(?:$|[?#])" pattern = BASE_PATTERN + r"/(?!photo/|liked)(?:p/)?([^/?#]+)/?(?:$|[?#])"
test = ( example = "https://500px.com/USER"
("https://500px.com/p/light_expression_photography", {
"pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2",
"range": "1-99",
"count": 99,
}),
("https://500px.com/light_expression_photography"),
("https://web.500px.com/light_expression_photography"),
)
def __init__(self, match): def __init__(self, match):
_500pxExtractor.__init__(self, match) _500pxExtractor.__init__(self, match)
@ -134,17 +126,7 @@ class _500pxGalleryExtractor(_500pxExtractor):
directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}") directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}")
pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?" pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?"
r"([^/?#]+)/galleries/([^/?#]+)") r"([^/?#]+)/galleries/([^/?#]+)")
test = ( example = "https://500px.com/USER/galleries/GALLERY"
("https://500px.com/p/fashvamp/galleries/lera", {
"url": "002dc81dee5b4a655f0e31ad8349e8903b296df6",
"count": 3,
"keyword": {
"gallery": dict,
"user": dict,
},
}),
("https://500px.com/fashvamp/galleries/lera"),
)
def __init__(self, match): def __init__(self, match):
_500pxExtractor.__init__(self, match) _500pxExtractor.__init__(self, match)
@ -200,7 +182,7 @@ class _500pxFavoriteExtractor(_500pxExtractor):
"""Extractor for favorite 500px photos""" """Extractor for favorite 500px photos"""
subcategory = "favorite" subcategory = "favorite"
pattern = BASE_PATTERN + r"/liked/?$" pattern = BASE_PATTERN + r"/liked/?$"
test = ("https://500px.com/liked",) example = "https://500px.com/liked"
def photos(self): def photos(self):
variables = {"pageSize": 20} variables = {"pageSize": 20}
@ -224,50 +206,7 @@ class _500pxImageExtractor(_500pxExtractor):
"""Extractor for individual images from 500px.com""" """Extractor for individual images from 500px.com"""
subcategory = "image" subcategory = "image"
pattern = BASE_PATTERN + r"/photo/(\d+)" pattern = BASE_PATTERN + r"/photo/(\d+)"
test = ("https://500px.com/photo/222049255/queen-of-coasts", { example = "https://500px.com/photo/12345/TITLE"
"url": "fbdf7df39325cae02f5688e9f92935b0e7113315",
"count": 1,
"keyword": {
"camera": "Canon EOS 600D",
"camera_info": dict,
"comments": list,
"comments_count": int,
"created_at": "2017-08-01T08:40:05+00:00",
"description": str,
"editored_by": None,
"editors_choice": False,
"extension": "jpg",
"feature": "popular",
"feature_date": "2017-08-01T09:58:28+00:00",
"focal_length": "208",
"height": 3111,
"id": 222049255,
"image_format": "jpg",
"image_url": list,
"images": list,
"iso": "100",
"lens": "EF-S55-250mm f/4-5.6 IS II",
"lens_info": dict,
"liked": None,
"location": None,
"location_details": dict,
"name": "Queen Of Coasts",
"nsfw": False,
"privacy": False,
"profile": True,
"rating": float,
"status": 1,
"tags": list,
"taken_at": "2017-05-04T17:36:51+00:00",
"times_viewed": int,
"url": "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva",
"user": dict,
"user_id": 12847235,
"votes_count": int,
"watermark": True,
"width": 4637,
},
})
def __init__(self, match): def __init__(self, match):
_500pxExtractor.__init__(self, match) _500pxExtractor.__init__(self, match)

53
gallery_dl/extractor/8chan.py
View File

@ -57,48 +57,7 @@ class _8chanThreadExtractor(_8chanExtractor):
filename_fmt = "{postId}{num:?-//} {filename[:200]}.{extension}" filename_fmt = "{postId}{num:?-//} {filename[:200]}.{extension}"
archive_fmt = "{boardUri}_{postId}_{num}" archive_fmt = "{boardUri}_{postId}_{num}"
pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)"
test = ( example = "https://8chan.moe/a/res/12345.html"
("https://8chan.moe/vhs/res/4.html", {
"pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$",
"count": 14,
"keyword": {
"archived": False,
"autoSage": False,
"boardDescription": "Film and Cinema",
"boardMarkdown": None,
"boardName": "Movies",
"boardUri": "vhs",
"creation": r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z",
"cyclic": False,
"email": None,
"id": "re:^[0-9a-f]{6}$",
"locked": False,
"markdown": str,
"maxFileCount": 5,
"maxFileSize": "32.00 MB",
"maxMessageLength": 8001,
"message": str,
"mime": str,
"name": "Anonymous",
"num": int,
"originalName": str,
"path": r"re:/.media/[0-9a-f]{64}\.\w+$",
"pinned": False,
"postId": int,
"signedRole": None,
"size": int,
"threadId": 4,
"thumb": r"re:/.media/t_[0-9a-f]{64}$",
"uniquePosters": 9,
"usesCustomCss": True,
"usesCustomJs": False,
"?wsPort": 8880,
"?wssPort": 2087,
},
}),
("https://8chan.se/vhs/res/4.html"),
("https://8chan.cc/vhs/res/4.html"),
)
def __init__(self, match): def __init__(self, match):
_8chanExtractor.__init__(self, match) _8chanExtractor.__init__(self, match)
@ -137,15 +96,7 @@ class _8chanBoardExtractor(_8chanExtractor):
"""Extractor for 8chan boards""" """Extractor for 8chan boards"""
subcategory = "board" subcategory = "board"
pattern = BASE_PATTERN + r"/([^/?#]+)/(?:(\d+)\.html)?$" pattern = BASE_PATTERN + r"/([^/?#]+)/(?:(\d+)\.html)?$"
test = ( example = "https://8chan.moe/a/"
("https://8chan.moe/vhs/"),
("https://8chan.moe/vhs/2.html", {
"pattern": _8chanThreadExtractor.pattern,
"count": 23,
}),
("https://8chan.se/vhs/"),
("https://8chan.cc/vhs/"),
)
def __init__(self, match): def __init__(self, match):
_8chanExtractor.__init__(self, match) _8chanExtractor.__init__(self, match)

46
gallery_dl/extractor/8muses.py
View File

@ -22,51 +22,7 @@ class _8musesAlbumExtractor(Extractor):
root = "https://comics.8muses.com" root = "https://comics.8muses.com"
pattern = (r"(?:https?://)?(?:comics\.|www\.)?8muses\.com" pattern = (r"(?:https?://)?(?:comics\.|www\.)?8muses\.com"
r"(/comics/album/[^?#]+)(\?[^#]+)?") r"(/comics/album/[^?#]+)(\?[^#]+)?")
test = ( example = "https://comics.8muses.com/comics/album/PATH/TITLE"
("https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", {
"url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4",
"pattern": r"https://comics.8muses.com/image/fl/[\w-]+",
"keyword": {
"url" : str,
"hash" : str,
"page" : int,
"count": 6,
"album": {
"id" : 10467,
"title" : "Liar",
"path" : "Fakku Comics/mogg/Liar",
"parts" : ["Fakku Comics", "mogg", "Liar"],
"private": False,
"url" : "https://comics.8muses.com/comics"
"/album/Fakku-Comics/mogg/Liar",
"parent" : 10464,
"views" : int,
"likes" : int,
"date" : "dt:2018-07-10 00:00:00",
},
},
}),
("https://www.8muses.com/comics/album/Fakku-Comics/santa", {
"count": ">= 3",
"pattern": pattern,
"keyword": {
"url" : str,
"name" : str,
"private": False,
},
}),
# custom sorting
("https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", {
"count": ">= 70",
"keyword": {"name": r"re:^[R-Zr-z]"},
}),
# non-ASCII characters
(("https://comics.8muses.com/comics/album/Various-Authors/Chessire88"
"/From-Trainers-to-Pokmons"), {
"count": 2,
"keyword": {"name": "re:From Trainers to Pokémons"},
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

14
gallery_dl/extractor/adultempire.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2019 Mike Fährmann # Copyright 2019-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -18,17 +18,7 @@ class AdultempireGalleryExtractor(GalleryExtractor):
root = "https://www.adultempire.com" root = "https://www.adultempire.com"
pattern = (r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com" pattern = (r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com"
r"(/(\d+)/gallery\.html)") r"(/(\d+)/gallery\.html)")
test = ( example = "https://www.adultempire.com/12345/gallery.html"
("https://www.adultempire.com/5998/gallery.html", {
"range": "1",
"keyword": "5b3266e69801db0d78c22181da23bc102886e027",
"content": "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e",
}),
("https://www.adultdvdempire.com/5683/gallery.html", {
"url": "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d",
"keyword": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5",
}),
)
def __init__(self, match): def __init__(self, match):
GalleryExtractor.__init__(self, match) GalleryExtractor.__init__(self, match)

27
gallery_dl/extractor/architizer.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2021 Mike Fährmann # Copyright 2021-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -21,25 +21,7 @@ class ArchitizerProjectExtractor(GalleryExtractor):
filename_fmt = "{filename}.{extension}" filename_fmt = "{filename}.{extension}"
archive_fmt = "{gid}_{num}" archive_fmt = "{gid}_{num}"
pattern = r"(?:https?://)?architizer\.com/projects/([^/?#]+)" pattern = r"(?:https?://)?architizer\.com/projects/([^/?#]+)"
test = ("https://architizer.com/projects/house-lo/", { example = "https://architizer.com/projects/NAME/"
"pattern": r"https://architizer-prod\.imgix\.net/media/mediadata"
r"/uploads/.+\.jpg$",
"keyword": {
"count": 27,
"description": str,
"firm": "Atelier Lina Bellovicova",
"gid": "225496",
"location": "Czechia",
"num": int,
"size": "1000 sqft - 3000 sqft",
"slug": "house-lo",
"status": "Built",
"subcategory": "project",
"title": "House LO",
"type": "Residential › Private House",
"year": "2020",
},
})
def __init__(self, match): def __init__(self, match):
url = "{}/projects/{}/".format(self.root, match.group(1)) url = "{}/projects/{}/".format(self.root, match.group(1))
@ -80,10 +62,7 @@ class ArchitizerFirmExtractor(Extractor):
subcategory = "firm" subcategory = "firm"
root = "https://architizer.com" root = "https://architizer.com"
pattern = r"(?:https?://)?architizer\.com/firms/([^/?#]+)" pattern = r"(?:https?://)?architizer\.com/firms/([^/?#]+)"
test = ("https://architizer.com/firms/olson-kundig/", { example = "https://architizer.com/firms/NAME/"
"pattern": ArchitizerProjectExtractor.pattern,
"count": ">= 90",
})
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

85
gallery_dl/extractor/artstation.py
View File

@ -178,17 +178,7 @@ class ArtstationUserExtractor(ArtstationExtractor):
pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com" pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com"
r"/(?!artwork|projects|search)([^/?#]+)(?:/albums/all)?" r"/(?!artwork|projects|search)([^/?#]+)(?:/albums/all)?"
r"|((?!www)\w+)\.artstation\.com(?:/projects)?)/?$") r"|((?!www)\w+)\.artstation\.com(?:/projects)?)/?$")
test = ( example = "https://www.artstation.com/USER"
("https://www.artstation.com/sungchoi/", {
"pattern": r"https://\w+\.artstation\.com/p/assets/images"
r"/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+",
"range": "1-10",
"count": ">= 10",
}),
("https://www.artstation.com/sungchoi/albums/all/"),
("https://sungchoi.artstation.com/"),
("https://sungchoi.artstation.com/projects/"),
)
def projects(self): def projects(self):
url = "{}/users/{}/projects.json".format(self.root, self.user) url = "{}/users/{}/projects.json".format(self.root, self.user)
@ -205,15 +195,7 @@ class ArtstationAlbumExtractor(ArtstationExtractor):
pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com" pattern = (r"(?:https?://)?(?:(?:www\.)?artstation\.com"
r"/(?!artwork|projects|search)([^/?#]+)" r"/(?!artwork|projects|search)([^/?#]+)"
r"|((?!www)\w+)\.artstation\.com)/albums/(\d+)") r"|((?!www)\w+)\.artstation\.com)/albums/(\d+)")
test = ( example = "https://www.artstation.com/USER/albums/12345"
("https://www.artstation.com/huimeiye/albums/770899", {
"count": 2,
}),
("https://www.artstation.com/huimeiye/albums/770898", {
"exception": exception.NotFoundError,
}),
("https://huimeiye.artstation.com/albums/770899"),
)
def __init__(self, match): def __init__(self, match):
ArtstationExtractor.__init__(self, match) ArtstationExtractor.__init__(self, match)
@ -247,17 +229,7 @@ class ArtstationLikesExtractor(ArtstationExtractor):
archive_fmt = "f_{userinfo[id]}_{asset[id]}" archive_fmt = "f_{userinfo[id]}_{asset[id]}"
pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" pattern = (r"(?:https?://)?(?:www\.)?artstation\.com"
r"/(?!artwork|projects|search)([^/?#]+)/likes/?") r"/(?!artwork|projects|search)([^/?#]+)/likes/?")
test = ( example = "https://www.artstation.com/USER/likes"
("https://www.artstation.com/mikf/likes", {
"pattern": r"https://\w+\.artstation\.com/p/assets/images"
r"/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+",
"count": 6,
}),
# no likes
("https://www.artstation.com/sungchoi/likes", {
"count": 0,
}),
)
def projects(self): def projects(self):
url = "{}/users/{}/likes.json".format(self.root, self.user) url = "{}/users/{}/likes.json".format(self.root, self.user)
@ -274,14 +246,7 @@ class ArtstationChallengeExtractor(ArtstationExtractor):
pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" pattern = (r"(?:https?://)?(?:www\.)?artstation\.com"
r"/contests/[^/?#]+/challenges/(\d+)" r"/contests/[^/?#]+/challenges/(\d+)"
r"/?(?:\?sorting=([a-z]+))?") r"/?(?:\?sorting=([a-z]+))?")
test = ( example = "https://www.artstation.com/contests/NAME/challenges/12345"
("https://www.artstation.com/contests/thu-2017/challenges/20"),
(("https://www.artstation.com/contests/beyond-human"
"/challenges/23?sorting=winners"), {
"range": "1-30",
"count": 30,
}),
)
def __init__(self, match): def __init__(self, match):
ArtstationExtractor.__init__(self, match) ArtstationExtractor.__init__(self, match)
@ -327,10 +292,7 @@ class ArtstationSearchExtractor(ArtstationExtractor):
archive_fmt = "s_{search[query]}_{asset[id]}" archive_fmt = "s_{search[query]}_{asset[id]}"
pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com" pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com"
r"/search/?\?([^#]+)") r"/search/?\?([^#]+)")
test = ("https://www.artstation.com/search?query=ancient&sort_by=rank", { example = "https://www.artstation.com/search?query=QUERY"
"range": "1-20",
"count": 20,
})
def __init__(self, match): def __init__(self, match):
ArtstationExtractor.__init__(self, match) ArtstationExtractor.__init__(self, match)
@ -377,10 +339,7 @@ class ArtstationArtworkExtractor(ArtstationExtractor):
archive_fmt = "A_{asset[id]}" archive_fmt = "A_{asset[id]}"
pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com" pattern = (r"(?:https?://)?(?:\w+\.)?artstation\.com"
r"/artwork/?\?([^#]+)") r"/artwork/?\?([^#]+)")
test = ("https://www.artstation.com/artwork?sorting=latest", { example = "https://www.artstation.com/artwork?sorting=SORT"
"range": "1-20",
"count": 20,
})
def __init__(self, match): def __init__(self, match):
ArtstationExtractor.__init__(self, match) ArtstationExtractor.__init__(self, match)
@ -400,32 +359,7 @@ class ArtstationImageExtractor(ArtstationExtractor):
pattern = (r"(?:https?://)?(?:" pattern = (r"(?:https?://)?(?:"
r"(?:\w+\.)?artstation\.com/(?:artwork|projects|search)" r"(?:\w+\.)?artstation\.com/(?:artwork|projects|search)"
r"|artstn\.co/p)/(\w+)") r"|artstn\.co/p)/(\w+)")
test = ( example = "https://www.artstation.com/artwork/abcde"
("https://www.artstation.com/artwork/LQVJr", {
"pattern": r"https?://\w+\.artstation\.com/p/assets"
r"/images/images/008/760/279/4k/.+",
"content": "7b113871465fdc09d127adfdc2767d51cf45a7e9",
# SHA1 hash without _no_cache()
# "content": "44b80f9af36d40efc5a2668cdd11d36d6793bae9",
}),
# multiple images per project
("https://www.artstation.com/artwork/Db3dy", {
"count": 4,
}),
# embedded youtube video
("https://www.artstation.com/artwork/g4WPK", {
"range": "2",
"options": (("external", True),),
"pattern": "ytdl:https://www.youtube.com/embed/JNFfJtwwrU0",
}),
# 404 (#3016)
("https://www.artstation.com/artwork/3q3mXB", {
"count": 0,
}),
# alternate URL patterns
("https://sungchoi.artstation.com/projects/LQVJr"),
("https://artstn.co/p/LQVJr"),
)
def __init__(self, match): def __init__(self, match):
ArtstationExtractor.__init__(self, match) ArtstationExtractor.__init__(self, match)
@ -453,10 +387,7 @@ class ArtstationFollowingExtractor(ArtstationExtractor):
subcategory = "following" subcategory = "following"
pattern = (r"(?:https?://)?(?:www\.)?artstation\.com" pattern = (r"(?:https?://)?(?:www\.)?artstation\.com"
r"/(?!artwork|projects|search)([^/?#]+)/following") r"/(?!artwork|projects|search)([^/?#]+)/following")
test = ("https://www.artstation.com/sungchoi/following", { example = "https://www.artstation.com/USER/following"
"pattern": ArtstationUserExtractor.pattern,
"count": ">= 50",
})
def items(self): def items(self):
url = "{}/users/{}/following.json".format(self.root, self.user) url = "{}/users/{}/following.json".format(self.root, self.user)

52
gallery_dl/extractor/aryion.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2020-2022 Mike Fährmann # Copyright 2020-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -176,16 +176,7 @@ class AryionGalleryExtractor(AryionExtractor):
subcategory = "gallery" subcategory = "gallery"
categorytransfer = True categorytransfer = True
pattern = BASE_PATTERN + r"/(?:gallery/|user/|latest.php\?name=)([^/?#]+)" pattern = BASE_PATTERN + r"/(?:gallery/|user/|latest.php\?name=)([^/?#]+)"
test = ( example = "https://aryion.com/g4/gallery/USER"
("https://aryion.com/g4/gallery/jameshoward", {
"options": (("recursive", False),),
"pattern": r"https://aryion\.com/g4/data\.php\?id=\d+$",
"range": "48-52",
"count": 5,
}),
("https://aryion.com/g4/user/jameshoward"),
("https://aryion.com/g4/latest.php?name=jameshoward"),
)
def __init__(self, match): def __init__(self, match):
AryionExtractor.__init__(self, match) AryionExtractor.__init__(self, match)
@ -215,9 +206,7 @@ class AryionTagExtractor(AryionExtractor):
directory_fmt = ("{category}", "tags", "{search_tags}") directory_fmt = ("{category}", "tags", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}" archive_fmt = "t_{search_tags}_{id}"
pattern = BASE_PATTERN + r"/tags\.php\?([^#]+)" pattern = BASE_PATTERN + r"/tags\.php\?([^#]+)"
test = ("https://aryion.com/g4/tags.php?tag=star+wars&p=19", { example = "https://aryion.com/g4/tags.php?tag=TAG"
"count": ">= 5",
})
def _init(self): def _init(self):
self.params = text.parse_query(self.user) self.params = text.parse_query(self.user)
@ -235,40 +224,7 @@ class AryionPostExtractor(AryionExtractor):
"""Extractor for individual posts on eka's portal""" """Extractor for individual posts on eka's portal"""
subcategory = "post" subcategory = "post"
pattern = BASE_PATTERN + r"/view/(\d+)" pattern = BASE_PATTERN + r"/view/(\d+)"
test = ( example = "https://aryion.com/g4/view/12345"
("https://aryion.com/g4/view/510079", {
"url": "f233286fa5558c07ae500f7f2d5cb0799881450e",
"keyword": {
"artist" : "jameshoward",
"user" : "jameshoward",
"filename" : "jameshoward-510079-subscribestar_150",
"extension": "jpg",
"id" : 510079,
"width" : 1665,
"height" : 1619,
"size" : 784239,
"title" : "I'm on subscribestar now too!",
"description": r"re:Doesn't hurt to have a backup, right\?",
"tags" : ["Non-Vore", "subscribestar"],
"date" : "dt:2019-02-16 19:30:34",
"path" : [],
"views" : int,
"favorites": int,
"comments" : int,
"_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT",
},
}),
# x-folder (#694)
("https://aryion.com/g4/view/588928", {
"pattern": pattern,
"count": ">= 8",
}),
# x-comic-folder (#945)
("https://aryion.com/g4/view/537379", {
"pattern": pattern,
"count": 2,
}),
)
def posts(self): def posts(self):
post_id, self.user = self.user, None post_id, self.user = self.user, None

25
gallery_dl/extractor/bbc.py
View File

@ -23,18 +23,7 @@ class BbcGalleryExtractor(GalleryExtractor):
filename_fmt = "{num:>02}.{extension}" filename_fmt = "{num:>02}.{extension}"
archive_fmt = "{programme}_{num}" archive_fmt = "{programme}_{num}"
pattern = BASE_PATTERN + r"[^/?#]+(?!/galleries)(?:/[^/?#]+)?)$" pattern = BASE_PATTERN + r"[^/?#]+(?!/galleries)(?:/[^/?#]+)?)$"
test = ( example = "https://www.bbc.co.uk/programmes/PATH"
("https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", {
"pattern": r"https://ichef\.bbci\.co\.uk"
r"/images/ic/1920xn/\w+\.jpg",
"count": 37,
"keyword": {
"programme": "p084qtzs",
"path": ["BBC One", "Doctor Who", "The Timeless Children"],
},
}),
("https://www.bbc.co.uk/programmes/p084qtzs"),
)
def metadata(self, page): def metadata(self, page):
data = util.json_loads(text.extr( data = util.json_loads(text.extr(
@ -72,17 +61,7 @@ class BbcProgrammeExtractor(Extractor):
subcategory = "programme" subcategory = "programme"
root = "https://www.bbc.co.uk" root = "https://www.bbc.co.uk"
pattern = BASE_PATTERN + r"[^/?#]+/galleries)(?:/?\?page=(\d+))?" pattern = BASE_PATTERN + r"[^/?#]+/galleries)(?:/?\?page=(\d+))?"
test = ( example = "https://www.bbc.co.uk/programmes/ID/galleries"
("https://www.bbc.co.uk/programmes/b006q2x0/galleries", {
"pattern": BbcGalleryExtractor.pattern,
"range": "1-50",
"count": ">= 50",
}),
("https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=40", {
"pattern": BbcGalleryExtractor.pattern,
"count": ">= 100",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

52
gallery_dl/extractor/behance.py
View File

@ -84,43 +84,7 @@ class BehanceGalleryExtractor(BehanceExtractor):
filename_fmt = "{category}_{id}_{num:>02}.{extension}" filename_fmt = "{category}_{id}_{num:>02}.{extension}"
archive_fmt = "{id}_{num}" archive_fmt = "{id}_{num}"
pattern = r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)" pattern = r"(?:https?://)?(?:www\.)?behance\.net/gallery/(\d+)"
test = ( example = "https://www.behance.net/gallery/12345/TITLE"
("https://www.behance.net/gallery/17386197/A-Short-Story", {
"count": 2,
"url": "ab79bd3bef8d3ae48e6ac74fd995c1dfaec1b7d2",
"keyword": {
"id": 17386197,
"name": 're:"Hi". A short story about the important things ',
"owners": ["Place Studio", "Julio César Velazquez"],
"fields": ["Animation", "Character Design", "Directing"],
"tags": list,
"module": dict,
"date": "dt:2014-06-03 15:41:51",
},
}),
("https://www.behance.net/gallery/21324767/Nevada-City", {
"count": 6,
"url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d",
"keyword": {"owners": ["Alex Strohl"]},
}),
# 'media_collection' modules
("https://www.behance.net/gallery/88276087/Audi-R8-RWD", {
"count": 20,
"url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f",
"pattern": r"https://mir-s3-cdn-cf\.behance\.net/project_modules"
r"/source/[0-9a-f]+.[0-9a-f]+\.jpg"
}),
# 'video' modules (#1282)
("https://www.behance.net/gallery/101185577/COLCCI", {
"pattern": r"https://cdn-prod-ccv\.adobe\.com/\w+"
r"/rend/\w+_720\.mp4\?",
"count": 3,
}),
# mature content (#4417)
("https://www.behance.net/gallery/177464639/Kimori", {
"exception": exception.AuthorizationError,
}),
)
def __init__(self, match): def __init__(self, match):
BehanceExtractor.__init__(self, match) BehanceExtractor.__init__(self, match)
@ -210,10 +174,7 @@ class BehanceUserExtractor(BehanceExtractor):
subcategory = "user" subcategory = "user"
categorytransfer = True categorytransfer = True
pattern = r"(?:https?://)?(?:www\.)?behance\.net/([^/?#]+)/?$" pattern = r"(?:https?://)?(?:www\.)?behance\.net/([^/?#]+)/?$"
test = ("https://www.behance.net/alexstrohl", { example = "https://www.behance.net/USER"
"count": ">= 11",
"pattern": BehanceGalleryExtractor.pattern,
})
def __init__(self, match): def __init__(self, match):
BehanceExtractor.__init__(self, match) BehanceExtractor.__init__(self, match)
@ -223,7 +184,7 @@ class BehanceUserExtractor(BehanceExtractor):
endpoint = "GetProfileProjects" endpoint = "GetProfileProjects"
variables = { variables = {
"username": self.user, "username": self.user,
"after" : "MAo=", "after" : "MAo=", # "0" in base64
} }
while True: while True:
@ -241,10 +202,7 @@ class BehanceCollectionExtractor(BehanceExtractor):
subcategory = "collection" subcategory = "collection"
categorytransfer = True categorytransfer = True
pattern = r"(?:https?://)?(?:www\.)?behance\.net/collection/(\d+)" pattern = r"(?:https?://)?(?:www\.)?behance\.net/collection/(\d+)"
test = ("https://www.behance.net/collection/71340149/inspiration", { example = "https://www.behance.net/collection/12345/TITLE"
"count": ">= 150",
"pattern": BehanceGalleryExtractor.pattern,
})
def __init__(self, match): def __init__(self, match):
BehanceExtractor.__init__(self, match) BehanceExtractor.__init__(self, match)
@ -253,7 +211,7 @@ class BehanceCollectionExtractor(BehanceExtractor):
def galleries(self): def galleries(self):
endpoint = "GetMoodboardItemsAndRecommendations" endpoint = "GetMoodboardItemsAndRecommendations"
variables = { variables = {
"afterItem": "MAo=", "afterItem": "MAo=", # "0" in base64
"firstItem": 40, "firstItem": 40,
"id" : int(self.collection_id), "id" : int(self.collection_id),
"shouldGetItems" : True, "shouldGetItems" : True,

82
gallery_dl/extractor/blogger.py
View File

@ -95,59 +95,8 @@ class BloggerExtractor(Extractor):
class BloggerPostExtractor(BloggerExtractor): class BloggerPostExtractor(BloggerExtractor):
"""Extractor for a single blog post""" """Extractor for a single blog post"""
subcategory = "post" subcategory = "post"
pattern = BASE_PATTERN + r"(/\d{4}/\d\d/[^/?#]+\.html)" pattern = BASE_PATTERN + r"(/\d\d\d\d/\d\d/[^/?#]+\.html)"
test = ( example = "https://BLOG.blogspot.com/YYYY/MM/TITLE.html"
("https://julianbphotography.blogspot.com/2010/12/moon-rise.html", {
"url": "9928429fb62f712eb4de80f53625eccecc614aae",
"pattern": r"https://3.bp.blogspot.com/.*/s0/Icy-Moonrise-.*.jpg",
"keyword": {
"blog": {
"date" : "dt:2010-11-21 18:19:42",
"description": "",
"id" : "5623928067739466034",
"kind" : "blogger#blog",
"locale" : dict,
"name" : "Julian Bunker Photography",
"pages" : int,
"posts" : int,
"published" : "2010-11-21T10:19:42-08:00",
"updated" : str,
"url" : "http://julianbphotography.blogspot.com/",
},
"post": {
"author" : "Julian Bunker",
"content" : str,
"date" : "dt:2010-12-26 01:08:00",
"etag" : str,
"id" : "6955139236418998998",
"kind" : "blogger#post",
"published" : "2010-12-25T17:08:00-08:00",
"replies" : "0",
"title" : "Moon Rise",
"updated" : "2011-12-06T05:21:24-08:00",
"url" : "re:.+/2010/12/moon-rise.html$",
},
"num": int,
"url": str,
},
}),
("blogger:http://www.julianbunker.com/2010/12/moon-rise.html"),
# video (#587)
(("http://cfnmscenesinmovies.blogspot.com/2011/11/"
"cfnm-scene-jenna-fischer-in-office.html"), {
"pattern": r"https://.+\.googlevideo\.com/videoplayback",
}),
# image URLs with width/height (#1061)
# ("https://aaaninja.blogspot.com/2020/08/altera-boob-press-2.html", {
# "pattern": r"https://1.bp.blogspot.com/.+/s0/altera_.+png",
# }),
# new image domain (#2204)
(("https://randomthingsthroughmyletterbox.blogspot.com/2022/01"
"/bitter-flowers-by-gunnar-staalesen-blog.html"), {
"pattern": r"https://blogger.googleusercontent.com/img/a/.+=s0$",
"count": 8,
}),
)
def __init__(self, match): def __init__(self, match):
BloggerExtractor.__init__(self, match) BloggerExtractor.__init__(self, match)
@ -161,17 +110,7 @@ class BloggerBlogExtractor(BloggerExtractor):
"""Extractor for an entire Blogger blog""" """Extractor for an entire Blogger blog"""
subcategory = "blog" subcategory = "blog"
pattern = BASE_PATTERN + r"/?$" pattern = BASE_PATTERN + r"/?$"
test = ( example = "https://BLOG.blogspot.com/"
("https://julianbphotography.blogspot.com/", {
"range": "1-25",
"count": 25,
"pattern": r"https://\d\.bp\.blogspot\.com/.*/s0/[^.]+\.jpg",
}),
("blogger:https://www.kefblog.com.ng/", {
"range": "1-25",
"count": 25,
}),
)
def posts(self, blog): def posts(self, blog):
return self.api.blog_posts(blog["id"]) return self.api.blog_posts(blog["id"])
@ -181,12 +120,7 @@ class BloggerSearchExtractor(BloggerExtractor):
"""Extractor for Blogger search resuls""" """Extractor for Blogger search resuls"""
subcategory = "search" subcategory = "search"
pattern = BASE_PATTERN + r"/search/?\?q=([^&#]+)" pattern = BASE_PATTERN + r"/search/?\?q=([^&#]+)"
test = ( example = "https://BLOG.blogspot.com/search?q=QUERY"
("https://julianbphotography.blogspot.com/search?q=400mm", {
"count": "< 10",
"keyword": {"query": "400mm"},
}),
)
def __init__(self, match): def __init__(self, match):
BloggerExtractor.__init__(self, match) BloggerExtractor.__init__(self, match)
@ -203,13 +137,7 @@ class BloggerLabelExtractor(BloggerExtractor):
"""Extractor for Blogger posts by label""" """Extractor for Blogger posts by label"""
subcategory = "label" subcategory = "label"
pattern = BASE_PATTERN + r"/search/label/([^/?#]+)" pattern = BASE_PATTERN + r"/search/label/([^/?#]+)"
test = ( example = "https://BLOG.blogspot.com/search/label/LABEL"
("https://dmmagazine.blogspot.com/search/label/D%26D", {
"range": "1-25",
"count": 25,
"keyword": {"label": "D&D"},
}),
)
def __init__(self, match): def __init__(self, match):
BloggerExtractor.__init__(self, match) BloggerExtractor.__init__(self, match)

48
gallery_dl/extractor/bunkr.py
View File

@ -29,53 +29,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor):
category = "bunkr" category = "bunkr"
root = "https://bunkrr.su" root = "https://bunkrr.su"
pattern = r"(?:https?://)?(?:app\.)?bunkr+\.(?:la|[sr]u|is|to)/a/([^/?#]+)" pattern = r"(?:https?://)?(?:app\.)?bunkr+\.(?:la|[sr]u|is|to)/a/([^/?#]+)"
test = ( example = "https://bunkrr.su/a/ID"
("https://bunkrr.su/a/Lktg9Keq", {
"pattern": r"https://cdn\.bunkr\.ru/test-テスト-\"&>-QjgneIQv\.png",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
"keyword": {
"album_id": "Lktg9Keq",
"album_name": 'test テスト "&>',
"count": 1,
"filename": 'test-テスト-"&>-QjgneIQv',
"id": "QjgneIQv",
"name": 'test-テスト-"&>',
"num": int,
},
}),
# mp4 (#2239)
("https://app.bunkr.ru/a/ptRHaCn2", {
"pattern": r"https://media-files\.bunkr\.ru/_-RnHoW69L\.mp4",
"content": "80e61d1dbc5896ae7ef9a28734c747b28b320471",
}),
# cdn4
("https://bunkr.is/a/iXTTc1o2", {
"pattern": r"https://(cdn|media-files)4\.bunkr\.ru/",
"content": "da29aae371b7adc8c5ef8e6991b66b69823791e8",
"keyword": {
"album_id": "iXTTc1o2",
"album_name": "test2",
"album_size": "691.1 KB",
"count": 2,
"description": "072022",
"filename": "re:video-wFO9FtxG|image-sZrQUeOx",
"id": "re:wFO9FtxG|sZrQUeOx",
"name": "re:video|image",
"num": int,
},
}),
# cdn12 .ru TLD (#4147)
("https://bunkrr.su/a/j1G29CnD", {
"pattern": r"https://(cdn12.bunkr.ru|media-files12.bunkr.la)/\w+",
"count": 8,
}),
("https://bunkrr.su/a/Lktg9Keq"),
("https://bunkr.la/a/Lktg9Keq"),
("https://bunkr.su/a/Lktg9Keq"),
("https://bunkr.ru/a/Lktg9Keq"),
("https://bunkr.is/a/Lktg9Keq"),
("https://bunkr.to/a/Lktg9Keq"),
)
def fetch_album(self, album_id): def fetch_album(self, album_id):
# album metadata # album metadata

27
gallery_dl/extractor/catbox.py
View File

@ -21,22 +21,7 @@ class CatboxAlbumExtractor(GalleryExtractor):
directory_fmt = ("{category}", "{album_name} ({album_id})") directory_fmt = ("{category}", "{album_name} ({album_id})")
archive_fmt = "{album_id}_{filename}" archive_fmt = "{album_id}_{filename}"
pattern = r"(?:https?://)?(?:www\.)?catbox\.moe(/c/[^/?#]+)" pattern = r"(?:https?://)?(?:www\.)?catbox\.moe(/c/[^/?#]+)"
test = ( example = "https://catbox.moe/c/ID"
("https://catbox.moe/c/1igcbe", {
"url": "35866a88c29462814f103bc22ec031eaeb380f8a",
"content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32",
"pattern": r"https://files\.catbox\.moe/\w+\.\w{3}$",
"count": 3,
"keyword": {
"album_id": "1igcbe",
"album_name": "test",
"date": "dt:2022-08-18 00:00:00",
"description": "album test &>",
},
}),
("https://www.catbox.moe/c/cd90s1"),
("https://catbox.moe/c/w7tm47#"),
)
def metadata(self, page): def metadata(self, page):
extr = text.extract_from(page) extr = text.extract_from(page)
@ -62,15 +47,7 @@ class CatboxFileExtractor(Extractor):
subcategory = "file" subcategory = "file"
archive_fmt = "{filename}" archive_fmt = "{filename}"
pattern = r"(?:https?://)?(?:files|litter|de)\.catbox\.moe/([^/?#]+)" pattern = r"(?:https?://)?(?:files|litter|de)\.catbox\.moe/([^/?#]+)"
test = ( example = "https://files.catbox.moe/NAME.EXT"
("https://files.catbox.moe/8ih3y7.png", {
"pattern": r"^https://files\.catbox\.moe/8ih3y7\.png$",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
"count": 1,
}),
("https://litter.catbox.moe/t8v3n9.png"),
("https://de.catbox.moe/bjdmz1.jpg"),
)
def items(self): def items(self):
url = text.ensure_http_scheme(self.url) url = text.ensure_http_scheme(self.url)

16
gallery_dl/extractor/comicvine.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2021 Mike Fährmann # Copyright 2021-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -25,19 +25,7 @@ class ComicvineTagExtractor(BooruExtractor):
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = (r"(?:https?://)?comicvine\.gamespot\.com" pattern = (r"(?:https?://)?comicvine\.gamespot\.com"
r"(/([^/?#]+)/(\d+-\d+)/images/.*)") r"(/([^/?#]+)/(\d+-\d+)/images/.*)")
test = ( example = "https://comicvine.gamespot.com/TAG/ID/images/"
("https://comicvine.gamespot.com/jock/4040-5653/images/", {
"pattern": r"https://comicvine\.gamespot\.com/a/uploads"
r"/original/\d+/\d+/\d+-.+\.(jpe?g|png)",
"count": ">= 140",
}),
(("https://comicvine.gamespot.com/batman/4005-1699"
"/images/?tag=Fan%20Art%20%26%20Cosplay"), {
"pattern": r"https://comicvine\.gamespot\.com/a/uploads"
r"/original/\d+/\d+/\d+-.+",
"count": ">= 450",
}),
)
def __init__(self, match): def __init__(self, match):
BooruExtractor.__init__(self, match) BooruExtractor.__init__(self, match)

2
gallery_dl/extractor/common.py
View File

@ -34,8 +34,8 @@ class Extractor():
archive_fmt = "" archive_fmt = ""
cookies_domain = "" cookies_domain = ""
browser = None browser = None
example = ""
root = "" root = ""
test = None
request_interval = 0.0 request_interval = 0.0
request_interval_min = 0.0 request_interval_min = 0.0
request_timestamp = 0.0 request_timestamp = 0.0

27
gallery_dl/extractor/cyberdrop.py
View File

@ -14,32 +14,7 @@ class CyberdropAlbumExtractor(lolisafe.LolisafeAlbumExtractor):
category = "cyberdrop" category = "cyberdrop"
root = "https://cyberdrop.me" root = "https://cyberdrop.me"
pattern = r"(?:https?://)?(?:www\.)?cyberdrop\.(?:me|to)/a/([^/?#]+)" pattern = r"(?:https?://)?(?:www\.)?cyberdrop\.(?:me|to)/a/([^/?#]+)"
test = ( example = "https://cyberdrop.me/a/ID"
# images
("https://cyberdrop.me/a/keKRjm4t", {
"pattern": r"https://fs-\d+\.cyberdrop\.to/.*\.(jpg|png|webp)$",
"keyword": {
"album_id": "keKRjm4t",
"album_name": "Fate (SFW)",
"album_size": 150069254,
"count": 62,
"date": "dt:2020-06-18 13:14:20",
"description": "",
"id": r"re:\w{8}",
},
}),
# videos
("https://cyberdrop.to/a/l8gIAXVD", {
"pattern": r"https://fs-\d+\.cyberdrop\.to/.*\.mp4$",
"count": 31,
"keyword": {
"album_id": "l8gIAXVD",
"album_name": "Achelois17 videos",
"album_size": 652037121,
"date": "dt:2020-06-16 15:40:44",
},
}),
)
def fetch_album(self, album_id): def fetch_album(self, album_id):
url = self.root + "/a/" + self.album_id url = self.root + "/a/" + self.album_id

82
gallery_dl/extractor/danbooru.py
View File

@ -173,38 +173,7 @@ class DanbooruTagExtractor(DanbooruExtractor):
directory_fmt = ("{category}", "{search_tags}") directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}" archive_fmt = "t_{search_tags}_{id}"
pattern = BASE_PATTERN + r"/posts\?(?:[^&#]*&)*tags=([^&#]*)" pattern = BASE_PATTERN + r"/posts\?(?:[^&#]*&)*tags=([^&#]*)"
test = ( example = "https://danbooru.donmai.us/posts?tags=TAG"
("https://danbooru.donmai.us/posts?tags=bonocho", {
"content": "b196fb9f1668109d7774a0a82efea3ffdda07746",
}),
# test page transitions
("https://danbooru.donmai.us/posts?tags=mushishi", {
"count": ">= 300",
}),
# 'external' option (#1747)
("https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", {
"options": (("external", True),),
"pattern": r"https://i\.pximg\.net/img-original/img"
r"/2008/08/28/02/35/48/1476533_p0\.jpg",
}),
("https://booru.allthefallen.moe/posts?tags=yume_shokunin", {
"count": 12,
}),
("https://aibooru.online/posts?tags=center_frills&z=1", {
"pattern": r"https://cdn\.aibooru\.online/original"
r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+",
"count": ">= 3",
}),
("https://booru.borvar.art/posts?tags=chibi&z=1", {
"pattern": r"https://booru\.borvar\.art/data/original"
r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+",
"count": ">= 3",
}),
("https://hijiribe.donmai.us/posts?tags=bonocho"),
("https://sonohara.donmai.us/posts?tags=bonocho"),
("https://safebooru.donmai.us/posts?tags=bonocho"),
("https://safe.aibooru.online/posts?tags=center_frills"),
)
def __init__(self, match): def __init__(self, match):
DanbooruExtractor.__init__(self, match) DanbooruExtractor.__init__(self, match)
@ -238,21 +207,7 @@ class DanbooruPoolExtractor(DanbooruExtractor):
directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}") directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}")
archive_fmt = "p_{pool[id]}_{id}" archive_fmt = "p_{pool[id]}_{id}"
pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)"
test = ( example = "https://danbooru.donmai.us/pools/12345"
("https://danbooru.donmai.us/pools/7659", {
"content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99",
}),
("https://booru.allthefallen.moe/pools/9", {
"url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5",
"count": 6,
}),
("https://booru.borvar.art/pools/2", {
"url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d",
"count": 4,
}),
("https://aibooru.online/pools/1"),
("https://danbooru.donmai.us/pool/show/7659"),
)
def __init__(self, match): def __init__(self, match):
DanbooruExtractor.__init__(self, match) DanbooruExtractor.__init__(self, match)
@ -275,26 +230,7 @@ class DanbooruPostExtractor(DanbooruExtractor):
subcategory = "post" subcategory = "post"
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)"
test = ( example = "https://danbooru.donmai.us/posts/12345"
("https://danbooru.donmai.us/posts/294929", {
"content": "5e255713cbf0a8e0801dc423563c34d896bb9229",
"keyword": {"date": "dt:2008-08-12 04:46:05"},
}),
("https://danbooru.donmai.us/posts/3613024", {
"pattern": r"https?://.+\.zip$",
"options": (("ugoira", True),)
}),
("https://booru.allthefallen.moe/posts/22", {
"content": "21dda68e1d7e0a554078e62923f537d8e895cac8",
}),
("https://aibooru.online/posts/1", {
"content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9",
}),
("https://booru.borvar.art/posts/1487", {
"content": "91273ac1ea413a12be468841e2b5804656a50bff",
}),
("https://danbooru.donmai.us/post/show/294929"),
)
def __init__(self, match): def __init__(self, match):
DanbooruExtractor.__init__(self, match) DanbooruExtractor.__init__(self, match)
@ -315,17 +251,7 @@ class DanbooruPopularExtractor(DanbooruExtractor):
directory_fmt = ("{category}", "popular", "{scale}", "{date}") directory_fmt = ("{category}", "popular", "{scale}", "{date}")
archive_fmt = "P_{scale[0]}_{date}_{id}" archive_fmt = "P_{scale[0]}_{date}_{id}"
pattern = BASE_PATTERN + r"/(?:explore/posts/)?popular(?:\?([^#]*))?" pattern = BASE_PATTERN + r"/(?:explore/posts/)?popular(?:\?([^#]*))?"
test = ( example = "https://danbooru.donmai.us/explore/posts/popular"
("https://danbooru.donmai.us/explore/posts/popular"),
(("https://danbooru.donmai.us/explore/posts/popular"
"?date=2013-06-06&scale=week"), {
"range": "1-120",
"count": 120,
}),
("https://booru.allthefallen.moe/explore/posts/popular"),
("https://aibooru.online/explore/posts/popular"),
("https://booru.borvar.art/explore/posts/popular"),
)
def __init__(self, match): def __init__(self, match):
DanbooruExtractor.__init__(self, match) DanbooruExtractor.__init__(self, match)

6
gallery_dl/extractor/desktopography.py
View File

@ -23,7 +23,7 @@ class DesktopographySiteExtractor(DesktopographyExtractor):
"""Extractor for all desktopography exhibitions """ """Extractor for all desktopography exhibitions """
subcategory = "site" subcategory = "site"
pattern = BASE_PATTERN + r"/$" pattern = BASE_PATTERN + r"/$"
test = ("https://desktopography.net/",) example = "https://desktopography.net/"
def items(self): def items(self):
page = self.request(self.root).text page = self.request(self.root).text
@ -42,7 +42,7 @@ class DesktopographyExhibitionExtractor(DesktopographyExtractor):
"""Extractor for a yearly desktopography exhibition""" """Extractor for a yearly desktopography exhibition"""
subcategory = "exhibition" subcategory = "exhibition"
pattern = BASE_PATTERN + r"/exhibition-([^/?#]+)/" pattern = BASE_PATTERN + r"/exhibition-([^/?#]+)/"
test = ("https://desktopography.net/exhibition-2020/",) example = "https://desktopography.net/exhibition-2020/"
def __init__(self, match): def __init__(self, match):
DesktopographyExtractor.__init__(self, match) DesktopographyExtractor.__init__(self, match)
@ -71,7 +71,7 @@ class DesktopographyEntryExtractor(DesktopographyExtractor):
"""Extractor for all resolutions of a desktopography wallpaper""" """Extractor for all resolutions of a desktopography wallpaper"""
subcategory = "entry" subcategory = "entry"
pattern = BASE_PATTERN + r"/portfolios/([\w-]+)" pattern = BASE_PATTERN + r"/portfolios/([\w-]+)"
test = ("https://desktopography.net/portfolios/new-era/",) example = "https://desktopography.net/portfolios/NAME/"
def __init__(self, match): def __init__(self, match):
DesktopographyExtractor.__init__(self, match) DesktopographyExtractor.__init__(self, match)

387
gallery_dl/extractor/deviantart.py
View File

@ -440,18 +440,7 @@ class DeviantartUserExtractor(DeviantartExtractor):
"""Extractor for an artist's user profile""" """Extractor for an artist's user profile"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/?$" pattern = BASE_PATTERN + r"/?$"
test = ( example = "https://www.deviantart.com/USER"
("https://www.deviantart.com/shimoda7", {
"pattern": r"/shimoda7/gallery$",
}),
("https://www.deviantart.com/shimoda7", {
"options": (("include", "all"),),
"pattern": r"/shimoda7/"
r"(gallery(/scraps)?|posts(/statuses)?|favourites)$",
"count": 5,
}),
("https://shimoda7.deviantart.com/"),
)
def initialize(self): def initialize(self):
pass pass
@ -475,84 +464,7 @@ class DeviantartGalleryExtractor(DeviantartExtractor):
subcategory = "gallery" subcategory = "gallery"
archive_fmt = "g_{_username}_{index}.{extension}" archive_fmt = "g_{_username}_{index}.{extension}"
pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$" pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$"
test = ( example = "https://www.deviantart.com/USER/gallery/"
("https://www.deviantart.com/shimoda7/gallery/", {
"pattern": r"https://(images-)?wixmp-[^.]+\.wixmp\.com"
r"/f/.+/.+\.(jpg|png)\?token=.+",
"count": ">= 30",
"keyword": {
"allows_comments": bool,
"author": {
"type": "regular",
"usericon": str,
"userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
"username": "shimoda7",
},
"category_path": str,
"content": {
"filesize": int,
"height": int,
"src": str,
"transparency": bool,
"width": int,
},
"da_category": str,
"date": "type:datetime",
"deviationid": str,
"?download_filesize": int,
"extension": str,
"index": int,
"is_deleted": bool,
"is_downloadable": bool,
"is_favourited": bool,
"is_mature": bool,
"preview": {
"height": int,
"src": str,
"transparency": bool,
"width": int,
},
"published_time": int,
"stats": {
"comments": int,
"favourites": int,
},
"target": dict,
"thumbs": list,
"title": str,
"url": r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+",
"username": "shimoda7",
},
}),
# group
("https://www.deviantart.com/yakuzafc/gallery", {
"pattern": r"https://www.deviantart.com/yakuzafc/gallery"
r"/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/",
"count": ">= 15",
}),
# 'folders' option (#276)
("https://www.deviantart.com/justatest235723/gallery", {
"count": 3,
"options": (("metadata", 1), ("folders", 1), ("original", 0)),
"keyword": {
"description": str,
"folders": list,
"is_watching": bool,
"license": str,
"tags": list,
},
}),
("https://www.deviantart.com/shimoda8/gallery/", {
"exception": exception.NotFoundError,
}),
("https://www.deviantart.com/shimoda7/gallery"),
("https://www.deviantart.com/shimoda7/gallery/all"),
("https://www.deviantart.com/shimoda7/gallery/?catpath=/"),
("https://shimoda7.deviantart.com/gallery/"),
("https://shimoda7.deviantart.com/gallery/all/"),
("https://shimoda7.deviantart.com/gallery/?catpath=/"),
)
def deviations(self): def deviations(self):
if self.flat and not self.group: if self.flat and not self.group:
@ -567,32 +479,7 @@ class DeviantartFolderExtractor(DeviantartExtractor):
directory_fmt = ("{category}", "{username}", "{folder[title]}") directory_fmt = ("{category}", "{username}", "{folder[title]}")
archive_fmt = "F_{folder[uuid]}_{index}.{extension}" archive_fmt = "F_{folder[uuid]}_{index}.{extension}"
pattern = BASE_PATTERN + r"/gallery/([^/?#]+)/([^/?#]+)" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)/([^/?#]+)"
test = ( example = "https://www.deviantart.com/USER/gallery/12345/TITLE"
# user
("https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", {
"count": 5,
"options": (("original", False),),
}),
# group
("https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", {
"count": ">= 4",
"options": (("original", False),),
}),
# uuid
(("https://www.deviantart.com/shimoda7/gallery"
"/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc"), {
"count": 5,
"options": (("original", False),),
}),
# name starts with '_', special characters (#1451)
(("https://www.deviantart.com/justatest235723"
"/gallery/69302698/-test-b-c-d-e-f-"), {
"count": 1,
"options": (("original", False),),
}),
("https://shimoda7.deviantart.com/gallery/722019/Miscellaneous"),
("https://yakuzafc.deviantart.com/gallery/37412168/Crafts"),
)
def __init__(self, match): def __init__(self, match):
DeviantartExtractor.__init__(self, match) DeviantartExtractor.__init__(self, match)
@ -621,33 +508,7 @@ class DeviantartStashExtractor(DeviantartExtractor):
subcategory = "stash" subcategory = "stash"
archive_fmt = "{index}.{extension}" archive_fmt = "{index}.{extension}"
pattern = r"(?:https?://)?sta\.sh/([a-z0-9]+)" pattern = r"(?:https?://)?sta\.sh/([a-z0-9]+)"
test = ( example = "https://sta.sh/abcde"
("https://sta.sh/022c83odnaxc", {
"pattern": r"https://wixmp-[^.]+\.wixmp\.com"
r"/f/.+/.+\.png\?token=.+",
"content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f",
"count": 1,
}),
# multiple stash items
("https://sta.sh/21jf51j7pzl2", {
"options": (("original", False),),
"count": 4,
}),
# downloadable, but no "content" field (#307)
("https://sta.sh/024t4coz16mi", {
"pattern": r"https://wixmp-[^.]+\.wixmp\.com"
r"/f/.+/.+\.rar\?token=.+",
"count": 1,
}),
# mixed folders and images (#659)
("https://sta.sh/215twi387vfj", {
"options": (("original", False),),
"count": 4,
}),
("https://sta.sh/abcdefghijkl", {
"count": 0,
}),
)
skip = Extractor.skip skip = Extractor.skip
@ -692,21 +553,7 @@ class DeviantartFavoriteExtractor(DeviantartExtractor):
directory_fmt = ("{category}", "{username}", "Favourites") directory_fmt = ("{category}", "{username}", "Favourites")
archive_fmt = "f_{_username}_{index}.{extension}" archive_fmt = "f_{_username}_{index}.{extension}"
pattern = BASE_PATTERN + r"/favourites(?:/all|/?\?catpath=)?/?$" pattern = BASE_PATTERN + r"/favourites(?:/all|/?\?catpath=)?/?$"
test = ( example = "https://www.deviantart.com/USER/favourites/"
# (#271)
("https://www.deviantart.com/h3813067/favourites/", {
"options": (("metadata", True), ("flat", False)),
"count": 1,
}),
("https://www.deviantart.com/h3813067/favourites/", {
"content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e",
}),
("https://www.deviantart.com/h3813067/favourites/all"),
("https://www.deviantart.com/h3813067/favourites/?catpath=/"),
("https://h3813067.deviantart.com/favourites/"),
("https://h3813067.deviantart.com/favourites/all"),
("https://h3813067.deviantart.com/favourites/?catpath=/"),
)
def deviations(self): def deviations(self):
if self.flat: if self.flat:
@ -723,20 +570,7 @@ class DeviantartCollectionExtractor(DeviantartExtractor):
"{collection[title]}") "{collection[title]}")
archive_fmt = "C_{collection[uuid]}_{index}.{extension}" archive_fmt = "C_{collection[uuid]}_{index}.{extension}"
pattern = BASE_PATTERN + r"/favourites/([^/?#]+)/([^/?#]+)" pattern = BASE_PATTERN + r"/favourites/([^/?#]+)/([^/?#]+)"
test = ( example = "https://www.deviantart.com/USER/favourites/12345/TITLE"
(("https://www.deviantart.com/pencilshadings/favourites"
"/70595441/3D-Favorites"), {
"count": ">= 15",
"options": (("original", False),),
}),
(("https://www.deviantart.com/pencilshadings/favourites"
"/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites"), {
"count": ">= 15",
"options": (("original", False),),
}),
("https://pencilshadings.deviantart.com"
"/favourites/70595441/3D-Favorites"),
)
def __init__(self, match): def __init__(self, match):
DeviantartExtractor.__init__(self, match) DeviantartExtractor.__init__(self, match)
@ -767,24 +601,7 @@ class DeviantartJournalExtractor(DeviantartExtractor):
directory_fmt = ("{category}", "{username}", "Journal") directory_fmt = ("{category}", "{username}", "Journal")
archive_fmt = "j_{_username}_{index}.{extension}" archive_fmt = "j_{_username}_{index}.{extension}"
pattern = BASE_PATTERN + r"/(?:posts(?:/journals)?|journal)/?(?:\?.*)?$" pattern = BASE_PATTERN + r"/(?:posts(?:/journals)?|journal)/?(?:\?.*)?$"
test = ( example = "https://www.deviantart.com/USER/posts/journals/"
("https://www.deviantart.com/angrywhitewanker/posts/journals/", {
"url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44",
}),
("https://www.deviantart.com/angrywhitewanker/posts/journals/", {
"url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e",
"options": (("journals", "text"),),
}),
("https://www.deviantart.com/angrywhitewanker/posts/journals/", {
"count": 0,
"options": (("journals", "none"),),
}),
("https://www.deviantart.com/shimoda7/posts/"),
("https://www.deviantart.com/shimoda7/journal/"),
("https://www.deviantart.com/shimoda7/journal/?catpath=/"),
("https://shimoda7.deviantart.com/journal/"),
("https://shimoda7.deviantart.com/journal/?catpath=/"),
)
def deviations(self): def deviations(self):
return self.api.browse_user_journals(self.user, self.offset) return self.api.browse_user_journals(self.user, self.offset)
@ -797,45 +614,7 @@ class DeviantartStatusExtractor(DeviantartExtractor):
filename_fmt = "{category}_{index}_{title}_{date}.{extension}" filename_fmt = "{category}_{index}_{title}_{date}.{extension}"
archive_fmt = "S_{_username}_{index}.{extension}" archive_fmt = "S_{_username}_{index}.{extension}"
pattern = BASE_PATTERN + r"/posts/statuses" pattern = BASE_PATTERN + r"/posts/statuses"
test = ( example = "https://www.deviantart.com/USER/posts/statuses/"
("https://www.deviantart.com/t1na/posts/statuses", {
"count": 0,
}),
("https://www.deviantart.com/justgalym/posts/statuses", {
"count": 4,
"url": "bf4c44c0c60ff2648a880f4c3723464ad3e7d074",
}),
# shared deviation
("https://www.deviantart.com/justgalym/posts/statuses", {
"options": (("journals", "none"),),
"count": 1,
"pattern": r"https://images-wixmp-\w+\.wixmp\.com/f"
r"/[^/]+/[^.]+\.jpg\?token=",
}),
# shared sta.sh item
("https://www.deviantart.com/vanillaghosties/posts/statuses", {
"options": (("journals", "none"), ("original", False)),
"range": "5-",
"count": 1,
"keyword": {
"index" : int,
"index_base36": "re:^[0-9a-z]+$",
"url" : "re:^https://sta.sh",
},
}),
# "deleted" deviations in 'items'
("https://www.deviantart.com/AndrejSKalin/posts/statuses", {
"options": (("journals", "none"), ("original", 0),
("image-filter", "deviationid[:8] == '147C8B03'")),
"count": 2,
"archive": False,
"keyword": {"deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655"}
}),
("https://www.deviantart.com/justgalym/posts/statuses", {
"options": (("journals", "text"),),
"url": "c8744f7f733a3029116607b826321233c5ca452d",
}),
)
def deviations(self): def deviations(self):
for status in self.api.user_statuses(self.user, self.offset): for status in self.api.user_statuses(self.user, self.offset):
@ -899,19 +678,7 @@ class DeviantartPopularExtractor(DeviantartExtractor):
r"(?:deviations/?)?\?order=(popular-[^/?#]+)" r"(?:deviations/?)?\?order=(popular-[^/?#]+)"
r"|((?:[\w-]+/)*)(popular-[^/?#]+)" r"|((?:[\w-]+/)*)(popular-[^/?#]+)"
r")/?(?:\?([^#]*))?") r")/?(?:\?([^#]*))?")
test = ( example = "https://www.deviantart.com/popular-24-hours/"
("https://www.deviantart.com/?order=popular-all-time", {
"options": (("original", False),),
"range": "1-30",
"count": 30,
}),
("https://www.deviantart.com/popular-24-hours/?q=tree+house", {
"options": (("original", False),),
"range": "1-30",
"count": 30,
}),
("https://www.deviantart.com/artisan/popular-all-time/?q=tree"),
)
def __init__(self, match): def __init__(self, match):
DeviantartExtractor.__init__(self, match) DeviantartExtractor.__init__(self, match)
@ -956,11 +723,7 @@ class DeviantartTagExtractor(DeviantartExtractor):
directory_fmt = ("{category}", "Tags", "{search_tags}") directory_fmt = ("{category}", "Tags", "{search_tags}")
archive_fmt = "T_{search_tags}_{index}.{extension}" archive_fmt = "T_{search_tags}_{index}.{extension}"
pattern = r"(?:https?://)?www\.deviantart\.com/tag/([^/?#]+)" pattern = r"(?:https?://)?www\.deviantart\.com/tag/([^/?#]+)"
test = ("https://www.deviantart.com/tag/nature", { example = "https://www.deviantart.com/tag/TAG"
"options": (("original", False),),
"range": "1-30",
"count": 30,
})
def __init__(self, match): def __init__(self, match):
DeviantartExtractor.__init__(self, match) DeviantartExtractor.__init__(self, match)
@ -979,10 +742,7 @@ class DeviantartWatchExtractor(DeviantartExtractor):
subcategory = "watch" subcategory = "watch"
pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com" pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com"
r"/(?:watch/deviations|notifications/watch)()()") r"/(?:watch/deviations|notifications/watch)()()")
test = ( example = "https://www.deviantart.com/watch/deviations"
("https://www.deviantart.com/watch/deviations"),
("https://www.deviantart.com/notifications/watch"),
)
def deviations(self): def deviations(self):
return self.api.browse_deviantsyouwatch() return self.api.browse_deviantsyouwatch()
@ -992,7 +752,7 @@ class DeviantartWatchPostsExtractor(DeviantartExtractor):
"""Extractor for Posts from watched users""" """Extractor for Posts from watched users"""
subcategory = "watch-posts" subcategory = "watch-posts"
pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()" pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()"
test = ("https://www.deviantart.com/watch/posts",) example = "https://www.deviantart.com/watch/posts"
def deviations(self): def deviations(self):
return self.api.browse_posts_deviantsyouwatch() return self.api.browse_posts_deviantsyouwatch()
@ -1010,100 +770,7 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)"
r"(\d+)" # bare deviation ID without slug r"(\d+)" # bare deviation ID without slug
r"|(?:https?://)?fav\.me/d([0-9a-z]+)") # base36 r"|(?:https?://)?fav\.me/d([0-9a-z]+)") # base36
test = ( example = "https://www.deviantart.com/UsER/art/TITLE-12345"
(("https://www.deviantart.com/shimoda7/art/For-the-sake-10073852"), {
"options": (("original", 0),),
"content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e",
}),
("https://www.deviantart.com/zzz/art/zzz-1234567890", {
"exception": exception.NotFoundError,
}),
(("https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576"), {
"options": (("comments", True),),
"keyword": {"comments": list},
"pattern": r"https://wixmp-[^.]+\.wixmp\.com"
r"/f/.+/.+\.jpg\?token=.+",
}),
# wixmp URL rewrite
(("https://www.deviantart.com/citizenfresh/art/Hverarond-789295466"), {
"pattern": (r"https://images-wixmp-\w+\.wixmp\.com/f"
r"/[^/]+/[^.]+\.jpg\?token="),
}),
# GIF (#242)
(("https://www.deviantart.com/skatergators/art/COM-Moni-781571783"), {
"pattern": r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-"
r"8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-"
r"f30961955bec\.gif\?token=ey...",
}),
# Flash animation with GIF preview (#1731)
("https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", {
"pattern": r"https://wixmp-[^.]+\.wixmp\.com"
r"/f/.+/.+\.swf\?token=.+",
"keyword": {
"filename": "flash_comic_tutorial_by_yuumei-d3juatd",
"extension": "swf",
},
}),
# sta.sh URLs from description (#302)
(("https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498"), {
"options": (("extra", 1), ("original", 0)),
"pattern": DeviantartStashExtractor.pattern,
"range": "2-",
"count": 4,
}),
# sta.sh URL from deviation["text_content"]["body"]["features"]
(("https://www.deviantart.com"
"/cimar-wildehopps/art/Honorary-Vixen-859809305"), {
"options": (("extra", 1),),
"pattern": ("text:<!DOCTYPE html>\n|" +
DeviantartStashExtractor.pattern),
"count": 2,
}),
# journal
("https://www.deviantart.com/shimoda7/journal/ARTility-583755752", {
"url": "d34b2c9f873423e665a1b8ced20fcb75951694a3",
"pattern": "text:<!DOCTYPE html>\n",
}),
# journal-like post with isJournal == False (#419)
("https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", {
"url": "e2e0044bd255304412179b6118536dbd9bb3bb0e",
"pattern": "text:<!DOCTYPE html>\n",
}),
# /view/ URLs
("https://deviantart.com/view/904858796/", {
"content": "8770ec40ad1c1d60f6b602b16301d124f612948f",
}),
("http://www.deviantart.com/view/890672057", {
"content": "1497e13d925caeb13a250cd666b779a640209236",
}),
("https://www.deviantart.com/view/706871727", {
"content": "3f62ae0c2fca2294ac28e41888ea06bb37c22c65",
}),
("https://www.deviantart.com/view/1", {
"exception": exception.NotFoundError,
}),
# /deviation/ (#3558)
("https://www.deviantart.com/deviation/817215762"),
# fav.me (#3558)
("https://fav.me/ddijrpu", {
"count": 1,
}),
("https://fav.me/dddd", {
"exception": exception.NotFoundError,
}),
# old-style URLs
("https://shimoda7.deviantart.com"
"/art/For-the-sake-of-a-memory-10073852"),
("https://myria-moon.deviantart.com"
"/art/Aime-Moi-part-en-vadrouille-261986576"),
("https://zzz.deviantart.com/art/zzz-1234567890"),
# old /view/ URLs from the Wayback Machine
("https://www.deviantart.com/view.php?id=14864502"),
("http://www.deviantart.com/view-full.php?id=100842"),
("https://www.fxdeviantart.com/zzz/art/zzz-1234567890"),
("https://www.fxdeviantart.com/view/1234567890"),
)
skip = Extractor.skip skip = Extractor.skip
@ -1134,13 +801,7 @@ class DeviantartScrapsExtractor(DeviantartExtractor):
archive_fmt = "s_{_username}_{index}.{extension}" archive_fmt = "s_{_username}_{index}.{extension}"
cookies_domain = ".deviantart.com" cookies_domain = ".deviantart.com"
pattern = BASE_PATTERN + r"/gallery/(?:\?catpath=)?scraps\b" pattern = BASE_PATTERN + r"/gallery/(?:\?catpath=)?scraps\b"
test = ( example = "https://www.deviantart.com/USER/gallery/scraps"
("https://www.deviantart.com/shimoda7/gallery/scraps", {
"count": 12,
}),
("https://www.deviantart.com/shimoda7/gallery/?catpath=scraps"),
("https://shimoda7.deviantart.com/gallery/?catpath=scraps"),
)
def deviations(self): def deviations(self):
self.login() self.login()
@ -1158,11 +819,7 @@ class DeviantartSearchExtractor(DeviantartExtractor):
cookies_domain = ".deviantart.com" cookies_domain = ".deviantart.com"
pattern = (r"(?:https?://)?www\.deviantart\.com" pattern = (r"(?:https?://)?www\.deviantart\.com"
r"/search(?:/deviations)?/?\?([^#]+)") r"/search(?:/deviations)?/?\?([^#]+)")
test = ( example = "https://www.deviantart.com/search?q=QUERY"
("https://www.deviantart.com/search?q=tree"),
("https://www.deviantart.com/search/deviations?order=popular-1-week"),
)
skip = Extractor.skip skip = Extractor.skip
def __init__(self, match): def __init__(self, match):
@ -1213,13 +870,7 @@ class DeviantartGallerySearchExtractor(DeviantartExtractor):
archive_fmt = "g_{_username}_{index}.{extension}" archive_fmt = "g_{_username}_{index}.{extension}"
cookies_domain = ".deviantart.com" cookies_domain = ".deviantart.com"
pattern = BASE_PATTERN + r"/gallery/?\?(q=[^#]+)" pattern = BASE_PATTERN + r"/gallery/?\?(q=[^#]+)"
test = ( example = "https://www.deviantart.com/USER/gallery?q=QUERY"
("https://www.deviantart.com/shimoda7/gallery?q=memory", {
"options": (("original", 0),),
"content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e",
}),
("https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular"),
)
def __init__(self, match): def __init__(self, match):
DeviantartExtractor.__init__(self, match) DeviantartExtractor.__init__(self, match)
@ -1251,11 +902,7 @@ class DeviantartFollowingExtractor(DeviantartExtractor):
"""Extractor for user's watched users""" """Extractor for user's watched users"""
subcategory = "following" subcategory = "following"
pattern = BASE_PATTERN + "/about#watching$" pattern = BASE_PATTERN + "/about#watching$"
test = ("https://www.deviantart.com/shimoda7/about#watching", { example = "https://www.deviantart.com/USER/about#watching"
"pattern": DeviantartUserExtractor.pattern,
"range": "1-50",
"count": 50,
})
def items(self): def items(self):
eclipse_api = DeviantartEclipseAPI(self) eclipse_api = DeviantartEclipseAPI(self)

33
gallery_dl/extractor/directlink.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2017-2022 Mike Fährmann # Copyright 2017-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -20,36 +20,7 @@ class DirectlinkExtractor(Extractor):
pattern = (r"(?i)https?://(?P<domain>[^/?#]+)/(?P<path>[^?#]+\." pattern = (r"(?i)https?://(?P<domain>[^/?#]+)/(?P<path>[^?#]+\."
r"(?:jpe?g|jpe|png|gif|web[mp]|mp4|mkv|og[gmv]|opus))" r"(?:jpe?g|jpe|png|gif|web[mp]|mp4|mkv|og[gmv]|opus))"
r"(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?$") r"(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?$")
test = ( example = "https://en.wikipedia.org/static/images/project-logos/enwiki.png"
(("https://en.wikipedia.org/static/images/project-logos/enwiki.png"), {
"url": "18c5d00077332e98e53be9fed2ee4be66154b88d",
"keyword": "105770a3f4393618ab7b811b731b22663b5d3794",
}),
# empty path
(("https://example.org/file.webm"), {
"url": "2d807ed7059d1b532f1bb71dc24b510b80ff943f",
"keyword": "29dad729c40fb09349f83edafa498dba1297464a",
}),
# more complex example
("https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", {
"url": "6fb1061390f8aada3db01cb24b51797c7ee42b31",
"keyword": "3d7abc31d45ba324e59bc599c3b4862452d5f29c",
}),
# percent-encoded characters
("https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", {
"url": "2627e8140727fdf743f86fe18f69f99a052c9718",
"keyword": "831790fddda081bdddd14f96985ab02dc5b5341f",
}),
# upper case file extension (#296)
("https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw"
".JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpP"
"mZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG"),
# internationalized domain name
("https://räksmörgås.josefsson.org/raksmorgas.jpg", {
"url": "a65667f670b194afbd1e3ea5e7a78938d36747da",
"keyword": "fd5037fe86eebd4764e176cbaf318caec0f700be",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

34
gallery_dl/extractor/dynastyscans.py
View File

@ -43,18 +43,7 @@ class DynastyscansBase():
class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor): class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor):
"""Extractor for manga-chapters from dynasty-scans.com""" """Extractor for manga-chapters from dynasty-scans.com"""
pattern = BASE_PATTERN + r"(/chapters/[^/?#]+)" pattern = BASE_PATTERN + r"(/chapters/[^/?#]+)"
test = ( example = "https://dynasty-scans.com/chapters/NAME"
(("http://dynasty-scans.com/chapters/"
"hitoribocchi_no_oo_seikatsu_ch33"), {
"url": "dce64e8c504118f1ab4135c00245ea12413896cb",
"keyword": "b67599703c27316a2fe4f11c3232130a1904e032",
}),
(("http://dynasty-scans.com/chapters/"
"new_game_the_spinoff_special_13"), {
"url": "dbe5bbb74da2edcfb1832895a484e2a40bc8b538",
"keyword": "6b674eb3a274999153f6be044973b195008ced2f",
}),
)
def metadata(self, page): def metadata(self, page):
extr = text.extract_from(page) extr = text.extract_from(page)
@ -93,10 +82,7 @@ class DynastyscansMangaExtractor(DynastyscansBase, MangaExtractor):
chapterclass = DynastyscansChapterExtractor chapterclass = DynastyscansChapterExtractor
reverse = False reverse = False
pattern = BASE_PATTERN + r"(/series/[^/?#]+)" pattern = BASE_PATTERN + r"(/series/[^/?#]+)"
test = ("https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", { example = "https://dynasty-scans.com/series/NAME"
"pattern": DynastyscansChapterExtractor.pattern,
"count": ">= 100",
})
def chapters(self, page): def chapters(self, page):
return [ return [
@ -112,16 +98,7 @@ class DynastyscansSearchExtractor(DynastyscansBase, Extractor):
filename_fmt = "{image_id}.{extension}" filename_fmt = "{image_id}.{extension}"
archive_fmt = "i_{image_id}" archive_fmt = "i_{image_id}"
pattern = BASE_PATTERN + r"/images/?(?:\?([^#]+))?$" pattern = BASE_PATTERN + r"/images/?(?:\?([^#]+))?$"
test = ( example = "https://dynasty-scans.com/images?QUERY"
("https://dynasty-scans.com/images?with[]=4930&with[]=5211", {
"url": "22cf0fb64e12b29e79b0a3d26666086a48f9916a",
"keyword": "11cbc555a15528d25567977b8808e10369c4c3ee",
}),
("https://dynasty-scans.com/images", {
"range": "1",
"count": 1,
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
@ -150,10 +127,7 @@ class DynastyscansImageExtractor(DynastyscansSearchExtractor):
"""Extractor for individual images on dynasty-scans.com""" """Extractor for individual images on dynasty-scans.com"""
subcategory = "image" subcategory = "image"
pattern = BASE_PATTERN + r"/images/(\d+)" pattern = BASE_PATTERN + r"/images/(\d+)"
test = ("https://dynasty-scans.com/images/1245", { example = "https://dynasty-scans.com/images/12345"
"url": "15e54bd94148a07ed037f387d046c27befa043b2",
"keyword": "0d8976c2d6fbc9ed6aa712642631b96e456dc37f",
})
def images(self): def images(self):
return (self.query,) return (self.query,)

135
gallery_dl/extractor/e621.py
View File

@ -84,48 +84,13 @@ BASE_PATTERN = E621Extractor.update({
class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor): class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor):
"""Extractor for e621 posts from tag searches""" """Extractor for e621 posts from tag searches"""
pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)" pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)"
test = ( example = "https://e621.net/posts?tags=TAG"
("https://e621.net/posts?tags=anry", {
"url": "8021e5ea28d47c474c1ffc9bd44863c4d45700ba",
"content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58",
}),
("https://e621.net/post/index/1/anry"),
("https://e621.net/post?tags=anry"),
("https://e926.net/posts?tags=anry", {
"url": "12198b275c62ffe2de67cca676c8e64de80c425d",
"content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58",
}),
("https://e926.net/post/index/1/anry"),
("https://e926.net/post?tags=anry"),
("https://e6ai.net/posts?tags=anry"),
("https://e6ai.net/post/index/1/anry"),
("https://e6ai.net/post?tags=anry"),
)
class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor): class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor):
"""Extractor for e621 pools""" """Extractor for e621 pools"""
pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)"
test = ( example = "https://e621.net/pools/12345"
("https://e621.net/pools/73", {
"url": "1bd09a72715286a79eea3b7f09f51b3493eb579a",
"content": "91abe5d5334425d9787811d7f06d34c77974cd22",
}),
("https://e621.net/pool/show/73"),
("https://e926.net/pools/73", {
"url": "6936f1b6a18c5c25bee7cad700088dbc2503481b",
"content": "91abe5d5334425d9787811d7f06d34c77974cd22",
}),
("https://e926.net/pool/show/73"),
("https://e6ai.net/pools/3", {
"url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f",
}),
("https://e6ai.net/pool/show/3"),
)
def posts(self): def posts(self):
self.log.info("Fetching posts of pool %s", self.pool_id) self.log.info("Fetching posts of pool %s", self.pool_id)
@ -151,67 +116,7 @@ class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor):
class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor): class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor):
"""Extractor for single e621 posts""" """Extractor for single e621 posts"""
pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)"
test = ( example = "https://e621.net/posts/12345"
("https://e621.net/posts/535", {
"url": "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529",
"content": "66f46e96a893fba8e694c4e049b23c2acc9af462",
"keyword": {"date": "dt:2007-02-17 19:02:32"},
}),
("https://e621.net/posts/3181052", {
"options": (("metadata", "notes,pools"),),
"pattern": r"https://static\d\.e621\.net/data/c6/8c"
r"/c68cca0643890b615f75fb2719589bff\.png",
"keyword": {
"notes": [
{
"body": "Little Legends 2",
"created_at": "2022-05-16T13:58:38.877-04:00",
"creator_id": 517450,
"creator_name": "EeveeCuddler69",
"height": 475,
"id": 321296,
"is_active": True,
"post_id": 3181052,
"updated_at": "2022-05-16T13:59:02.050-04:00",
"version": 3,
"width": 809,
"x": 83,
"y": 117,
},
],
"pools": [
{
"category": "series",
"created_at": "2022-02-17T00:29:22.669-05:00",
"creator_id": 1077440,
"creator_name": "Yeetus90",
"description": "* \"Little Legends\":/pools/27971\r\n"
"* Little Legends 2\r\n"
"* \"Little Legends 3\":/pools/27481",
"id": 27492,
"is_active": False,
"name": "Little Legends 2",
"post_count": 39,
"post_ids": list,
"updated_at": "2022-03-27T06:30:03.382-04:00"
},
],
},
}),
("https://e621.net/post/show/535"),
("https://e926.net/posts/535", {
"url": "17aec8ebd8fab098d321adcb62a2db59dab1f4bf",
"content": "66f46e96a893fba8e694c4e049b23c2acc9af462",
}),
("https://e926.net/post/show/535"),
("https://e6ai.net/posts/23", {
"url": "3c85a806b3d9eec861948af421fe0e8ad6b8f881",
"content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5",
}),
("https://e6ai.net/post/show/23"),
)
def posts(self): def posts(self):
url = "{}/posts/{}.json".format(self.root, self.post_id) url = "{}/posts/{}.json".format(self.root, self.post_id)
@ -221,23 +126,7 @@ class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor):
class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor): class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor):
"""Extractor for popular images from e621""" """Extractor for popular images from e621"""
pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?" pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?"
test = ( example = "https://e621.net/explore/posts/popular"
("https://e621.net/explore/posts/popular"),
(("https://e621.net/explore/posts/popular"
"?date=2019-06-01&scale=month"), {
"pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+",
"count": ">= 70",
}),
("https://e926.net/explore/posts/popular"),
(("https://e926.net/explore/posts/popular"
"?date=2019-06-01&scale=month"), {
"pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+",
"count": ">= 70",
}),
("https://e6ai.net/explore/posts/popular"),
)
def posts(self): def posts(self):
return self._pagination("/popular.json", self.params) return self._pagination("/popular.json", self.params)
@ -249,21 +138,7 @@ class E621FavoriteExtractor(E621Extractor):
directory_fmt = ("{category}", "Favorites", "{user_id}") directory_fmt = ("{category}", "Favorites", "{user_id}")
archive_fmt = "f_{user_id}_{id}" archive_fmt = "f_{user_id}_{id}"
pattern = BASE_PATTERN + r"/favorites(?:\?([^#]*))?" pattern = BASE_PATTERN + r"/favorites(?:\?([^#]*))?"
test = ( example = "https://e621.net/favorites"
("https://e621.net/favorites"),
("https://e621.net/favorites?page=2&user_id=53275", {
"pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+",
"count": "> 260",
}),
("https://e926.net/favorites"),
("https://e926.net/favorites?page=2&user_id=53275", {
"pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+",
"count": "> 260",
}),
("https://e6ai.net/favorites"),
)
def __init__(self, match): def __init__(self, match):
E621Extractor.__init__(self, match) E621Extractor.__init__(self, match)

34
gallery_dl/extractor/erome.py
View File

@ -91,29 +91,7 @@ class EromeAlbumExtractor(EromeExtractor):
"""Extractor for albums on erome.com""" """Extractor for albums on erome.com"""
subcategory = "album" subcategory = "album"
pattern = BASE_PATTERN + r"/a/(\w+)" pattern = BASE_PATTERN + r"/a/(\w+)"
test = ( example = "https://www.erome.com/a/ID"
("https://www.erome.com/a/NQgdlWvk", {
"pattern": r"https://v\d+\.erome\.com/\d+"
r"/NQgdlWvk/j7jlzmYB_480p\.mp4",
"count": 1,
"keyword": {
"album_id": "NQgdlWvk",
"num": 1,
"title": "porn",
"user": "yYgWBZw8o8qsMzM",
},
}),
("https://www.erome.com/a/TdbZ4ogi", {
"pattern": r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+",
"count": 6,
"keyword": {
"album_id": "TdbZ4ogi",
"num": int,
"title": "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.",
"user": "yYgWBZw8o8qsMzM",
},
}),
)
def albums(self): def albums(self):
return (self.item,) return (self.item,)
@ -122,10 +100,7 @@ class EromeAlbumExtractor(EromeExtractor):
class EromeUserExtractor(EromeExtractor): class EromeUserExtractor(EromeExtractor):
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/(?!a/|search\?)([^/?#]+)" pattern = BASE_PATTERN + r"/(?!a/|search\?)([^/?#]+)"
test = ("https://www.erome.com/yYgWBZw8o8qsMzM", { example = "https://www.erome.com/USER"
"range": "1-25",
"count": 25,
})
def albums(self): def albums(self):
url = "{}/{}".format(self.root, self.item) url = "{}/{}".format(self.root, self.item)
@ -135,10 +110,7 @@ class EromeUserExtractor(EromeExtractor):
class EromeSearchExtractor(EromeExtractor): class EromeSearchExtractor(EromeExtractor):
subcategory = "search" subcategory = "search"
pattern = BASE_PATTERN + r"/search\?q=([^&#]+)" pattern = BASE_PATTERN + r"/search\?q=([^&#]+)"
test = ("https://www.erome.com/search?q=cute", { example = "https://www.erome.com/search?q=QUERY"
"range": "1-25",
"count": 25,
})
def albums(self): def albums(self):
url = self.root + "/search" url = self.root + "/search"

82
gallery_dl/extractor/exhentai.py
View File

@ -109,61 +109,7 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
pattern = (BASE_PATTERN + pattern = (BASE_PATTERN +
r"(?:/g/(\d+)/([\da-f]{10})" r"(?:/g/(\d+)/([\da-f]{10})"
r"|/s/([\da-f]{10})/(\d+)-(\d+))") r"|/s/([\da-f]{10})/(\d+)-(\d+))")
test = ( example = "https://e-hentai.org/g/12345/67890abcde/"
("https://exhentai.org/g/1200119/d55c44d3d0/", {
"options": (("original", False),),
"keyword": {
"cost": int,
"date": "dt:2018-03-18 20:14:00",
"eh_category": "Non-H",
"expunged": False,
"favorites": r"re:^[12]\d$",
"filecount": "4",
"filesize": 1488978,
"gid": 1200119,
"height": int,
"image_token": "re:[0-9a-f]{10}",
"lang": "ja",
"language": "Japanese",
"parent": "",
"rating": r"re:\d\.\d+",
"size": int,
"tags": [
"parody:komi-san wa komyushou desu.",
"character:shouko komi",
"group:seventh lowlife",
"other:sample",
],
"thumb": "https://exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f8"
"3bcb1630ab1350640-624622-736-1036-jpg_250.jpg",
"title": "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan de"
"su (Komi-san wa Komyushou desu) [Sample]",
"title_jpn": "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆"
"です。 (古見さんは、コミュ症です。) [見本]",
"token": "d55c44d3d0",
"torrentcount": "0",
"uploader": "klorpa",
"width": int,
},
"content": ("2c68cff8a7ca540a78c36fdbf5fbae0260484f87",
"e9891a4c017ed0bb734cd1efba5cd03f594d31ff"),
}),
("https://exhentai.org/g/960461/4f0e369d82/", {
"exception": exception.NotFoundError,
}),
("http://exhentai.org/g/962698/7f02358e00/", {
"exception": exception.AuthorizationError,
}),
("https://exhentai.org/s/f68367b4c8/1200119-3", {
"options": (("original", False),),
"count": 2,
}),
("https://e-hentai.org/s/f68367b4c8/1200119-3", {
"options": (("original", False),),
"count": 2,
}),
("https://g.e-hentai.org/g/1200119/d55c44d3d0/"),
)
def __init__(self, match): def __init__(self, match):
ExhentaiExtractor.__init__(self, match) ExhentaiExtractor.__init__(self, match)
@ -481,22 +427,7 @@ class ExhentaiSearchExtractor(ExhentaiExtractor):
"""Extractor for exhentai search results""" """Extractor for exhentai search results"""
subcategory = "search" subcategory = "search"
pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))" pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))"
test = ( example = "https://e-hentai.org/?f_search=QUERY"
("https://e-hentai.org/?f_search=touhou"),
("https://exhentai.org/?f_cats=767&f_search=touhou"),
("https://exhentai.org/tag/parody:touhou+project"),
(("https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0"
"&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0"
"&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter"), {
"pattern": ExhentaiGalleryExtractor.pattern,
"range": "1-30",
"count": 30,
"keyword": {
"gallery_id": int,
"gallery_token": r"re:^[0-9a-f]{10}$"
},
}),
)
def __init__(self, match): def __init__(self, match):
ExhentaiExtractor.__init__(self, match) ExhentaiExtractor.__init__(self, match)
@ -553,14 +484,7 @@ class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor):
"""Extractor for favorited exhentai galleries""" """Extractor for favorited exhentai galleries"""
subcategory = "favorite" subcategory = "favorite"
pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?" pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?"
test = ( example = "https://e-hentai.org/favorites.php"
("https://e-hentai.org/favorites.php", {
"count": 1,
"pattern": r"https?://e-hentai\.org/g/1200119/d55c44d3d0"
}),
("https://exhentai.org/favorites.php?favcat=1&f_search=touhou"
"&f_apply=Search+Favorites"),
)
def _init(self): def _init(self):
self.search_url = self.root + "/favorites.php" self.search_url = self.root + "/favorites.php"

30
gallery_dl/extractor/fallenangels.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2017-2019 Mike Fährmann # Copyright 2017-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -13,24 +13,11 @@ from .. import text, util
class FallenangelsChapterExtractor(ChapterExtractor): class FallenangelsChapterExtractor(ChapterExtractor):
"""Extractor for manga-chapters from fascans.com""" """Extractor for manga chapters from fascans.com"""
category = "fallenangels" category = "fallenangels"
pattern = (r"(?:https?://)?(manga|truyen)\.fascans\.com" pattern = (r"(?:https?://)?(manga|truyen)\.fascans\.com"
r"/manga/([^/?#]+)/([^/?#]+)") r"/manga/([^/?#]+)/([^/?#]+)")
test = ( example = "https://manga.fascans.com/manga/NAME/CHAPTER/"
("https://manga.fascans.com/manga/chronos-ruler/20/1", {
"url": "4604a7914566cc2da0ff789aa178e2d1c8c241e3",
"keyword": "2dfcc50020e32cd207be88e2a8fac0933e36bdfb",
}),
("http://truyen.fascans.com/manga/hungry-marie/8", {
"url": "1f923d9cb337d5e7bbf4323719881794a951c6ae",
"keyword": "2bdb7334c0e3eceb9946ffd3132df679b4a94f6a",
}),
("http://manga.fascans.com/manga/rakudai-kishi-no-eiyuutan/19.5", {
"url": "273f6863966c83ea79ad5846a2866e08067d3f0e",
"keyword": "d1065685bfe0054c4ff2a0f20acb089de4cec253",
}),
)
def __init__(self, match): def __init__(self, match):
self.version, self.manga, self.chapter = match.groups() self.version, self.manga, self.chapter = match.groups()
@ -66,16 +53,7 @@ class FallenangelsMangaExtractor(MangaExtractor):
chapterclass = FallenangelsChapterExtractor chapterclass = FallenangelsChapterExtractor
category = "fallenangels" category = "fallenangels"
pattern = r"(?:https?://)?((manga|truyen)\.fascans\.com/manga/[^/]+)/?$" pattern = r"(?:https?://)?((manga|truyen)\.fascans\.com/manga/[^/]+)/?$"
test = ( example = "https://manga.fascans.com/manga/NAME"
("https://manga.fascans.com/manga/chronos-ruler", {
"url": "eea07dd50f5bc4903aa09e2cc3e45c7241c9a9c2",
"keyword": "c414249525d4c74ad83498b3c59a813557e59d7e",
}),
("https://truyen.fascans.com/manga/rakudai-kishi-no-eiyuutan", {
"url": "51a731a6b82d5eb7a335fbae6b02d06aeb2ab07b",
"keyword": "2d2a2a5d9ea5925eb9a47bb13d848967f3af086c",
}),
)
def __init__(self, match): def __init__(self, match):
url = "https://" + match.group(1) url = "https://" + match.group(1)

72
gallery_dl/extractor/fanbox.py
View File

@ -10,7 +10,6 @@ from .common import Extractor, Message
from .. import text from .. import text
import re import re
BASE_PATTERN = ( BASE_PATTERN = (
r"(?:https?://)?(?:" r"(?:https?://)?(?:"
r"(?!www\.)([\w-]+)\.fanbox\.cc|" r"(?!www\.)([\w-]+)\.fanbox\.cc|"
@ -30,12 +29,12 @@ class FanboxExtractor(Extractor):
def _init(self): def _init(self):
self.embeds = self.config("embeds", True) self.embeds = self.config("embeds", True)
def items(self):
if self._warning: if self._warning:
if not self.cookies_check(("FANBOXSESSID",)): if not self.cookies_check(("FANBOXSESSID",)):
self.log.warning("no 'FANBOXSESSID' cookie set") self.log.warning("no 'FANBOXSESSID' cookie set")
FanboxExtractor._warning = False FanboxExtractor._warning = False
def items(self):
for content_body, post in self.posts(): for content_body, post in self.posts():
yield Message.Directory, post yield Message.Directory, post
yield from self._get_urls_from_post(content_body, post) yield from self._get_urls_from_post(content_body, post)
@ -243,20 +242,7 @@ class FanboxCreatorExtractor(FanboxExtractor):
"""Extractor for a Fanbox creator's works""" """Extractor for a Fanbox creator's works"""
subcategory = "creator" subcategory = "creator"
pattern = BASE_PATTERN + r"(?:/posts)?/?$" pattern = BASE_PATTERN + r"(?:/posts)?/?$"
test = ( example = "https://USER.fanbox.cc/"
("https://xub.fanbox.cc", {
"range": "1-15",
"count": ">= 15",
"keyword": {
"creatorId" : "xub",
"tags" : list,
"title" : str,
},
}),
("https://xub.fanbox.cc/posts"),
("https://www.fanbox.cc/@xub/"),
("https://www.fanbox.cc/@xub/posts"),
)
def __init__(self, match): def __init__(self, match):
FanboxExtractor.__init__(self, match) FanboxExtractor.__init__(self, match)
@ -271,55 +257,7 @@ class FanboxPostExtractor(FanboxExtractor):
"""Extractor for media from a single Fanbox post""" """Extractor for media from a single Fanbox post"""
subcategory = "post" subcategory = "post"
pattern = BASE_PATTERN + r"/posts/(\d+)" pattern = BASE_PATTERN + r"/posts/(\d+)"
test = ( example = "https://USER.fanbox.cc/posts/12345"
("https://www.fanbox.cc/@xub/posts/1910054", {
"count": 3,
"keyword": {
"title": "えま★おうがすと",
"tags": list,
"hasAdultContent": True,
"isCoverImage": False
},
}),
# entry post type, image embedded in html of the post
("https://nekoworks.fanbox.cc/posts/915", {
"count": 2,
"keyword": {
"title": "【SAYORI FAN CLUB】お届け内容",
"tags": list,
"html": str,
"hasAdultContent": True
},
}),
# article post type, imageMap, 2 twitter embeds, fanbox embed
("https://steelwire.fanbox.cc/posts/285502", {
"options": (("embeds", True),),
"count": 10,
"keyword": {
"title": "イラスト+SS｜義足の炭鉱少年が義足を見せてくれるだけ 【全体公開版】",
"tags": list,
"articleBody": dict,
"hasAdultContent": True
},
}),
# 'content' metadata (#3020)
("https://www.fanbox.cc/@official-en/posts/4326303", {
"keyword": {
"content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, "
r"September 5th, 2022, we are happy to announce "
r"the start of the FANBOX hashtag event "
r"#MySetupTour ! \nAbout the event\nTo join this "
r"event .+ \nPlease check this page for further "
r"details regarding the Privacy & Terms.\n"
r"https://fanbox.pixiv.help/.+/10184952456601\n\n\n"
r"Thank you for your continued support of FANBOX.$",
},
}),
# imageMap file order (#2718)
("https://mochirong.fanbox.cc/posts/3746116", {
"url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062",
}),
)
def __init__(self, match): def __init__(self, match):
FanboxExtractor.__init__(self, match) FanboxExtractor.__init__(self, match)
@ -334,9 +272,7 @@ class FanboxRedirectExtractor(Extractor):
category = "fanbox" category = "fanbox"
subcategory = "redirect" subcategory = "redirect"
pattern = r"(?:https?://)?(?:www\.)?pixiv\.net/fanbox/creator/(\d+)" pattern = r"(?:https?://)?(?:www\.)?pixiv\.net/fanbox/creator/(\d+)"
test = ("https://www.pixiv.net/fanbox/creator/52336352", { example = "https://www.pixiv.net/fanbox/creator/12345"
"pattern": FanboxCreatorExtractor.pattern,
})
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

50
gallery_dl/extractor/fanleaks.py
View File

@ -7,7 +7,7 @@
"""Extractors for https://fanleaks.club/""" """Extractors for https://fanleaks.club/"""
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, exception from .. import text
class FanleaksExtractor(Extractor): class FanleaksExtractor(Extractor):
@ -36,34 +36,10 @@ class FanleaksExtractor(Extractor):
class FanleaksPostExtractor(FanleaksExtractor): class FanleaksPostExtractor(FanleaksExtractor):
"""Extractor for individual posts on fanleak.club""" """Extractor for individual posts on fanleaks.club"""
subcategory = "post" subcategory = "post"
pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club/([^/?#]+)/(\d+)" pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club/([^/?#]+)/(\d+)"
test = ( example = "https://fanleaks.club/MODEL/12345"
("https://fanleaks.club/selti/880", {
"pattern": (r"https://fanleaks\.club//models"
r"/selti/images/selti_0880\.jpg"),
"keyword": {
"model_id": "selti",
"model" : "Selti",
"id" : 880,
"type" : "photo",
},
}),
("https://fanleaks.club/daisy-keech/1038", {
"pattern": (r"https://fanleaks\.club//models"
r"/daisy-keech/videos/daisy-keech_1038\.mp4"),
"keyword": {
"model_id": "daisy-keech",
"model" : "Daisy Keech",
"id" : 1038,
"type" : "video",
},
}),
("https://fanleaks.club/hannahowo/000", {
"exception": exception.NotFoundError,
}),
)
def __init__(self, match): def __init__(self, match):
FanleaksExtractor.__init__(self, match) FanleaksExtractor.__init__(self, match)
@ -79,22 +55,7 @@ class FanleaksModelExtractor(FanleaksExtractor):
subcategory = "model" subcategory = "model"
pattern = (r"(?:https?://)?(?:www\.)?fanleaks\.club" pattern = (r"(?:https?://)?(?:www\.)?fanleaks\.club"
r"/(?!latest/?$)([^/?#]+)/?$") r"/(?!latest/?$)([^/?#]+)/?$")
test = ( example = "https://fanleaks.club/MODEL"
("https://fanleaks.club/hannahowo", {
"pattern": (r"https://fanleaks\.club//models"
r"/hannahowo/(images|videos)/hannahowo_\d+\.\w+"),
"range" : "1-100",
"count" : 100,
}),
("https://fanleaks.club/belle-delphine", {
"pattern": (r"https://fanleaks\.club//models"
r"/belle-delphine/(images|videos)"
r"/belle-delphine_\d+\.\w+"),
"range" : "1-100",
"count" : 100,
}),
("https://fanleaks.club/daisy-keech"),
)
def items(self): def items(self):
page_num = 1 page_num = 1
@ -102,8 +63,7 @@ class FanleaksModelExtractor(FanleaksExtractor):
self.root + "/" + self.model_id, notfound="model").text self.root + "/" + self.model_id, notfound="model").text
data = { data = {
"model_id": self.model_id, "model_id": self.model_id,
"model" : text.unescape( "model" : text.unescape(text.extr(page, 'mt-4">', "</h1>")),
text.extr(page, 'mt-4">', "</h1>")),
"type" : "photo", "type" : "photo",
} }
page_url = text.extr(page, "url: '", "'") page_url = text.extr(page, "url: '", "'")

60
gallery_dl/extractor/fantia.py
View File

@ -173,17 +173,7 @@ class FantiaCreatorExtractor(FantiaExtractor):
"""Extractor for a Fantia creator's works""" """Extractor for a Fantia creator's works"""
subcategory = "creator" subcategory = "creator"
pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/fanclubs/(\d+)" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/fanclubs/(\d+)"
test = ( example = "https://fantia.jp/fanclubs/12345"
("https://fantia.jp/fanclubs/6939", {
"range": "1-25",
"count": ">= 25",
"keyword": {
"fanclub_user_id" : 52152,
"tags" : list,
"title" : str,
},
}),
)
def __init__(self, match): def __init__(self, match):
FantiaExtractor.__init__(self, match) FantiaExtractor.__init__(self, match)
@ -198,53 +188,7 @@ class FantiaPostExtractor(FantiaExtractor):
"""Extractor for media from a single Fantia post""" """Extractor for media from a single Fantia post"""
subcategory = "post" subcategory = "post"
pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/posts/(\d+)" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/posts/(\d+)"
test = ( example = "https://fantia.jp/posts/12345"
("https://fantia.jp/posts/1166373", {
"pattern": r"https://("
r"c\.fantia\.jp/uploads/post/file/1166373/|"
r"cc\.fantia\.jp/uploads/post_content_photo"
r"/file/732549[01]|"
r"fantia\.jp/posts/1166373/album_image\?)",
"keyword": {
"blogpost_text": r"re:^$|"
r"This is a test.\n\nThis is a test.\n\n|"
r"Link to video:\nhttps://www.youtube.com"
r"/watch\?v=5SSdvNcAagI\n\nhtml img from "
r"another site:\n\n\n\n\n\n",
"comment": "\n\n",
"content_category": "re:thumb|blog|photo_gallery",
"content_comment": str,
"content_filename": "re:|",
"content_title": r"re:Test (Blog Content \d+|Image Gallery)"
r"|thumb",
"date": "dt:2022-03-09 16:46:12",
"fanclub_id": 356320,
"fanclub_name": "Test Fantia",
"fanclub_url": "https://fantia.jp/fanclubs/356320",
"fanclub_user_id": 7487131,
"fanclub_user_name": "2022/03/08 15:13:52の名無し",
"file_url": str,
"filename": str,
"num": int,
"plan": dict,
"post_id": 1166373,
"post_title": "Test Fantia Post",
"post_url": "https://fantia.jp/posts/1166373",
"posted_at": "Thu, 10 Mar 2022 01:46:12 +0900",
"rating": "general",
"tags": [],
},
}),
("https://fantia.jp/posts/508363", {
"count": 6,
"keyword": {
"post_title": "zunda逆バニーでおしりｺｯｼｮﾘ",
"tags": list,
"rating": "adult",
"post_id": 508363
},
}),
)
def __init__(self, match): def __init__(self, match):
FantiaExtractor.__init__(self, match) FantiaExtractor.__init__(self, match)

27
gallery_dl/extractor/fapachi.py
View File

@ -14,25 +14,13 @@ class FapachiPostExtractor(Extractor):
"""Extractor for individual posts on fapachi.com""" """Extractor for individual posts on fapachi.com"""
category = "fapachi" category = "fapachi"
subcategory = "post" subcategory = "post"
root = "https://fapachi.com"
directory_fmt = ("{category}", "{user}") directory_fmt = ("{category}", "{user}")
filename_fmt = "{user}_{id}.{extension}" filename_fmt = "{user}_{id}.{extension}"
archive_fmt = "{user}_{id}" archive_fmt = "{user}_{id}"
pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com"
r"/(?!search/)([^/?#]+)/media/(\d+)") r"/(?!search/)([^/?#]+)/media/(\d+)")
root = "https://fapachi.com" example = "https://fapachi.com/MODEL/media/12345"
test = (
# NSFW
("https://fapachi.com/sonson/media/0082", {
"pattern": (r"https://fapachi\.com/models/s/o/"
r"sonson/1/full/sonson_0082\.jpeg"),
"keyword": {
"user": "sonson",
"id" : "0082",
},
}),
# NSFW
("https://fapachi.com/ferxiita/media/0159"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
@ -54,17 +42,10 @@ class FapachiUserExtractor(Extractor):
"""Extractor for all posts from a fapachi user""" """Extractor for all posts from a fapachi user"""
category = "fapachi" category = "fapachi"
subcategory = "user" subcategory = "user"
root = "https://fapachi.com"
pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com"
r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$") r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$")
root = "https://fapachi.com" example = "https://fapachi.com/MODEL"
test = (
("https://fapachi.com/sonson", {
"pattern": FapachiPostExtractor.pattern,
"range" : "1-50",
"count" : 50,
}),
("https://fapachi.com/ferxiita/page/3"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

53
gallery_dl/extractor/fapello.py
View File

@ -19,32 +19,7 @@ class FapelloPostExtractor(Extractor):
archive_fmt = "{type}_{model}_{id}" archive_fmt = "{type}_{model}_{id}"
pattern = (r"(?:https?://)?(?:www\.)?fapello\.com" pattern = (r"(?:https?://)?(?:www\.)?fapello\.com"
r"/(?!search/|popular_videos/)([^/?#]+)/(\d+)") r"/(?!search/|popular_videos/)([^/?#]+)/(\d+)")
test = ( example = "https://fapello.com/MODEL/12345/"
("https://fapello.com/carrykey/530/", {
"pattern": (r"https://fapello\.com/content/c/a"
r"/carrykey/1000/carrykey_0530\.jpg"),
"keyword": {
"model": "carrykey",
"id" : 530,
"type" : "photo",
"thumbnail": "",
},
}),
("https://fapello.com/vladislava-661/693/", {
"pattern": (r"https://cdn\.fapello\.com/content/v/l"
r"/vladislava-661/1000/vladislava-661_0693\.mp4"),
"keyword": {
"model": "vladislava-661",
"id" : 693,
"type" : "video",
"thumbnail": ("https://fapello.com/content/v/l"
"/vladislava-661/1000/vladislava-661_0693.jpg"),
},
}),
("https://fapello.com/carrykey/000/", {
"exception": exception.NotFoundError,
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
@ -77,14 +52,7 @@ class FapelloModelExtractor(Extractor):
r"/(?!top-(?:likes|followers)|popular_videos" r"/(?!top-(?:likes|followers)|popular_videos"
r"|videos|trending|search/?$)" r"|videos|trending|search/?$)"
r"([^/?#]+)/?$") r"([^/?#]+)/?$")
test = ( example = "https://fapello.com/model/"
("https://fapello.com/hyoon/", {
"pattern": FapelloPostExtractor.pattern,
"range" : "1-50",
"count" : 50,
}),
("https://fapello.com/kobaebeefboo/"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
@ -112,22 +80,7 @@ class FapelloPathExtractor(Extractor):
pattern = (r"(?:https?://)?(?:www\.)?fapello\.com" pattern = (r"(?:https?://)?(?:www\.)?fapello\.com"
r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" r"/(?!search/?$)(top-(?:likes|followers)|videos|trending"
r"|popular_videos/[^/?#]+)/?$") r"|popular_videos/[^/?#]+)/?$")
test = ( example = "https://fapello.com/PATH/"
("https://fapello.com/top-likes/", {
"pattern": FapelloModelExtractor.pattern,
"range" : "1-10",
"count" : 10,
}),
("https://fapello.com/videos/", {
"pattern": FapelloPostExtractor.pattern,
"range" : "1-10",
"count" : 10,
}),
("https://fapello.com/top-followers/"),
("https://fapello.com/trending/"),
("https://fapello.com/popular_videos/twelve_hours/"),
("https://fapello.com/popular_videos/week/"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

66
gallery_dl/extractor/flickr.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2017-2022 Mike Fährmann # Copyright 2017-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -64,42 +64,7 @@ class FlickrImageExtractor(FlickrExtractor):
r"(?:(?:www\.|secure\.|m\.)?flickr\.com/photos/[^/?#]+/" r"(?:(?:www\.|secure\.|m\.)?flickr\.com/photos/[^/?#]+/"
r"|[\w-]+\.static\.?flickr\.com/(?:\d+/)+)(\d+)" r"|[\w-]+\.static\.?flickr\.com/(?:\d+/)+)(\d+)"
r"|flic\.kr/p/([A-Za-z1-9]+))") r"|flic\.kr/p/([A-Za-z1-9]+))")
test = ( example = "https://www.flickr.com/photos/USER/12345"
("https://www.flickr.com/photos/departingyyz/16089302239", {
"pattern": pattern,
"content": ("3133006c6d657fe54cf7d4c46b82abbcb0efaf9f",
"0821a28ee46386e85b02b67cf2720063440a228c"),
"keyword": {
"comments": int,
"description": str,
"extension": "jpg",
"filename": "16089302239_de18cd8017_b",
"id": 16089302239,
"height": 683,
"label": "Large",
"media": "photo",
"url": str,
"views": int,
"width": 1024,
},
}),
("https://secure.flickr.com/photos/departingyyz/16089302239"),
("https://m.flickr.com/photos/departingyyz/16089302239"),
("https://flickr.com/photos/departingyyz/16089302239"),
("https://www.flickr.com/photos/145617051@N08/46733161535", {
"count": 1,
"keyword": {"media": "video"},
}),
("http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", {
"pattern": pattern}),
("https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", {
"pattern": pattern}),
("https://flic.kr/p/FPVo9U", {
"pattern": pattern}),
("https://www.flickr.com/photos/zzz/16089302238", {
"exception": exception.NotFoundError}),
)
def __init__(self, match): def __init__(self, match):
FlickrExtractor.__init__(self, match) FlickrExtractor.__init__(self, match)
@ -194,11 +159,7 @@ class FlickrGalleryExtractor(FlickrExtractor):
"Galleries", "{gallery[gallery_id]} {gallery[title]}") "Galleries", "{gallery[gallery_id]} {gallery[title]}")
archive_fmt = "g_{gallery[id]}_{id}" archive_fmt = "g_{gallery[id]}_{id}"
pattern = BASE_PATTERN + r"/photos/([^/?#]+)/galleries/(\d+)" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/galleries/(\d+)"
test = (("https://www.flickr.com/photos/flickr/" example = "https://www.flickr.com/photos/USER/galleries/12345/"
"galleries/72157681572514792/"), {
"pattern": FlickrImageExtractor.pattern,
"count": ">= 10",
})
def __init__(self, match): def __init__(self, match):
FlickrExtractor.__init__(self, match) FlickrExtractor.__init__(self, match)
@ -219,10 +180,7 @@ class FlickrGroupExtractor(FlickrExtractor):
directory_fmt = ("{category}", "Groups", "{group[groupname]}") directory_fmt = ("{category}", "Groups", "{group[groupname]}")
archive_fmt = "G_{group[nsid]}_{id}" archive_fmt = "G_{group[nsid]}_{id}"
pattern = BASE_PATTERN + r"/groups/([^/?#]+)" pattern = BASE_PATTERN + r"/groups/([^/?#]+)"
test = ("https://www.flickr.com/groups/bird_headshots/", { example = "https://www.flickr.com/groups/NAME/"
"pattern": FlickrImageExtractor.pattern,
"count": "> 150",
})
def metadata(self): def metadata(self):
self.group = self.api.urls_lookupGroup(self.item_id) self.group = self.api.urls_lookupGroup(self.item_id)
@ -237,10 +195,7 @@ class FlickrUserExtractor(FlickrExtractor):
subcategory = "user" subcategory = "user"
archive_fmt = "u_{user[nsid]}_{id}" archive_fmt = "u_{user[nsid]}_{id}"
pattern = BASE_PATTERN + r"/photos/([^/?#]+)/?$" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/?$"
test = ("https://www.flickr.com/photos/shona_s/", { example = "https://www.flickr.com/photos/USER/"
"pattern": FlickrImageExtractor.pattern,
"count": 28,
})
def photos(self): def photos(self):
return self.api.people_getPhotos(self.user["nsid"]) return self.api.people_getPhotos(self.user["nsid"])
@ -252,10 +207,7 @@ class FlickrFavoriteExtractor(FlickrExtractor):
directory_fmt = ("{category}", "{user[username]}", "Favorites") directory_fmt = ("{category}", "{user[username]}", "Favorites")
archive_fmt = "f_{user[nsid]}_{id}" archive_fmt = "f_{user[nsid]}_{id}"
pattern = BASE_PATTERN + r"/photos/([^/?#]+)/favorites" pattern = BASE_PATTERN + r"/photos/([^/?#]+)/favorites"
test = ("https://www.flickr.com/photos/shona_s/favorites", { example = "https://www.flickr.com/photos/USER/favorites"
"pattern": FlickrImageExtractor.pattern,
"count": 4,
})
def photos(self): def photos(self):
return self.api.favorites_getList(self.user["nsid"]) return self.api.favorites_getList(self.user["nsid"])
@ -267,11 +219,7 @@ class FlickrSearchExtractor(FlickrExtractor):
directory_fmt = ("{category}", "Search", "{search[text]}") directory_fmt = ("{category}", "Search", "{search[text]}")
archive_fmt = "s_{search}_{id}" archive_fmt = "s_{search}_{id}"
pattern = BASE_PATTERN + r"/search/?\?([^#]+)" pattern = BASE_PATTERN + r"/search/?\?([^#]+)"
test = ( example = "https://flickr.com/search/?text=QUERY"
("https://flickr.com/search/?text=mountain"),
("https://flickr.com/search/?text=tree%20cloud%20house"
"&color_codes=4&styles=minimalism"),
)
def __init__(self, match): def __init__(self, match):
FlickrExtractor.__init__(self, match) FlickrExtractor.__init__(self, match)

74
gallery_dl/extractor/foolfuuka.py
View File

@ -111,43 +111,7 @@ class FoolfuukaThreadExtractor(FoolfuukaExtractor):
directory_fmt = ("{category}", "{board[shortname]}", directory_fmt = ("{category}", "{board[shortname]}",
"{thread_num} {title|comment[:50]}") "{thread_num} {title|comment[:50]}")
pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)" pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)"
test = ( example = "https://archived.moe/a/thread/12345/"
("https://archive.4plebs.org/tg/thread/54059290", {
"url": "fd823f17b5001442b941fddcd9ec91bafedfbc79",
}),
("https://archived.moe/gd/thread/309639/", {
"url": "fdd533840e2d535abd162c02d6dfadbc12e2dcd8",
"content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573",
}),
("https://archived.moe/a/thread/159767162/", {
"url": "ffec05a1a1b906b5ca85992513671c9155ee9e87",
}),
("https://archiveofsins.com/h/thread/4668813/", {
"url": "f612d287087e10a228ef69517cf811539db9a102",
"content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4",
}),
("https://arch.b4k.co/meta/thread/196/", {
"url": "d309713d2f838797096b3e9cb44fe514a9c9d07a",
}),
("https://desuarchive.org/a/thread/159542679/", {
"url": "e7d624aded15a069194e38dc731ec23217a422fb",
}),
("https://boards.fireden.net/sci/thread/11264294/", {
"url": "61cab625c95584a12a30049d054931d64f8d20aa",
}),
("https://archive.palanq.win/c/thread/4209598/", {
"url": "1f9b5570d228f1f2991c827a6631030bc0e5933c",
}),
("https://rbt.asia/g/thread/61487650/", {
"url": "fadd274b25150a1bdf03a40c58db320fa3b617c4",
}),
("https://archive.rebeccablacktech.com/g/thread/61487650/", {
"url": "fadd274b25150a1bdf03a40c58db320fa3b617c4",
}),
("https://thebarchive.com/b/thread/739772332/", {
"url": "e8b18001307d130d67db31740ce57c8561b5d80c",
}),
)
def __init__(self, match): def __init__(self, match):
FoolfuukaExtractor.__init__(self, match) FoolfuukaExtractor.__init__(self, match)
@ -175,17 +139,7 @@ class FoolfuukaBoardExtractor(FoolfuukaExtractor):
"""Base extractor for FoolFuuka based boards/archives""" """Base extractor for FoolFuuka based boards/archives"""
subcategory = "board" subcategory = "board"
pattern = BASE_PATTERN + r"/([^/?#]+)/\d*$" pattern = BASE_PATTERN + r"/([^/?#]+)/\d*$"
test = ( example = "https://archived.moe/a/"
("https://archive.4plebs.org/tg/"),
("https://archived.moe/gd/"),
("https://archiveofsins.com/h/"),
("https://arch.b4k.co/meta/"),
("https://desuarchive.org/a/"),
("https://boards.fireden.net/sci/"),
("https://archive.palanq.win/c/"),
("https://rbt.asia/g/"),
("https://thebarchive.com/b/"),
)
def __init__(self, match): def __init__(self, match):
FoolfuukaExtractor.__init__(self, match) FoolfuukaExtractor.__init__(self, match)
@ -217,18 +171,8 @@ class FoolfuukaSearchExtractor(FoolfuukaExtractor):
subcategory = "search" subcategory = "search"
directory_fmt = ("{category}", "search", "{search}") directory_fmt = ("{category}", "search", "{search}")
pattern = BASE_PATTERN + r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)" pattern = BASE_PATTERN + r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)"
example = "https://archived.moe/_/search/text/QUERY/"
request_interval = 1.0 request_interval = 1.0
test = (
("https://archive.4plebs.org/_/search/text/test/"),
("https://archived.moe/_/search/text/test/"),
("https://archiveofsins.com/_/search/text/test/"),
("https://archiveofsins.com/_/search/text/test/"),
("https://desuarchive.org/_/search/text/test/"),
("https://boards.fireden.net/_/search/text/test/"),
("https://archive.palanq.win/_/search/text/test/"),
("https://rbt.asia/_/search/text/test/"),
("https://thebarchive.com/_/search/text/test/"),
)
def __init__(self, match): def __init__(self, match):
FoolfuukaExtractor.__init__(self, match) FoolfuukaExtractor.__init__(self, match)
@ -283,17 +227,7 @@ class FoolfuukaGalleryExtractor(FoolfuukaExtractor):
subcategory = "gallery" subcategory = "gallery"
directory_fmt = ("{category}", "{board}", "gallery") directory_fmt = ("{category}", "{board}", "gallery")
pattern = BASE_PATTERN + r"/([^/?#]+)/gallery(?:/(\d+))?" pattern = BASE_PATTERN + r"/([^/?#]+)/gallery(?:/(\d+))?"
test = ( example = "https://archived.moe/a/gallery"
("https://archive.4plebs.org/tg/gallery/1"),
("https://archived.moe/gd/gallery/2"),
("https://archiveofsins.com/h/gallery/3"),
("https://arch.b4k.co/meta/gallery/"),
("https://desuarchive.org/a/gallery/5"),
("https://boards.fireden.net/sci/gallery/6"),
("https://archive.palanq.win/c/gallery"),
("https://rbt.asia/g/gallery/8"),
("https://thebarchive.com/b/gallery/9"),
)
def __init__(self, match): def __init__(self, match):
FoolfuukaExtractor.__init__(self, match) FoolfuukaExtractor.__init__(self, match)

26
gallery_dl/extractor/foolslide.py
View File

@ -53,13 +53,7 @@ class FoolslideChapterExtractor(FoolslideExtractor):
"{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}")
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = BASE_PATTERN + r"(/read/[^/?#]+/[a-z-]+/\d+/\d+(?:/\d+)?)" pattern = BASE_PATTERN + r"(/read/[^/?#]+/[a-z-]+/\d+/\d+(?:/\d+)?)"
test = ( example = "https://read.powermanga.org/read/MANGA/en/0/CHAPTER/"
(("https://read.powermanga.org"
"/read/one_piece_digital_colour_comics/en/0/75/"), {
"url": "854c5817f8f767e1bccd05fa9d58ffb5a4b09384",
"keyword": "a60c42f2634b7387899299d411ff494ed0ad6dbe",
}),
)
def items(self): def items(self):
page = self.request(self.gallery_url).text page = self.request(self.gallery_url).text
@ -103,23 +97,7 @@ class FoolslideMangaExtractor(FoolslideExtractor):
subcategory = "manga" subcategory = "manga"
categorytransfer = True categorytransfer = True
pattern = BASE_PATTERN + r"(/series/[^/?#]+)" pattern = BASE_PATTERN + r"(/series/[^/?#]+)"
test = ( example = "https://read.powermanga.org/series/MANGA/"
(("https://read.powermanga.org"
"/series/one_piece_digital_colour_comics/"), {
"count": ">= 1",
"keyword": {
"chapter": int,
"chapter_minor": str,
"chapter_string": str,
"group": "PowerManga",
"lang": "en",
"language": "English",
"manga": "One Piece Digital Colour Comics",
"title": str,
"volume": int,
},
}),
)
def items(self): def items(self):
page = self.request(self.gallery_url).text page = self.request(self.gallery_url).text

118
gallery_dl/extractor/furaffinity.py
View File

@ -31,6 +31,7 @@ class FuraffinityExtractor(Extractor):
def _init(self): def _init(self):
self.offset = 0 self.offset = 0
self.external = self.config("external", False)
if self.config("descriptions") == "html": if self.config("descriptions") == "html":
self._process_description = str.strip self._process_description = str.strip
@ -41,13 +42,12 @@ class FuraffinityExtractor(Extractor):
else: else:
self._new_layout = None self._new_layout = None
def items(self):
if self._warning: if self._warning:
if not self.cookies_check(self.cookies_names): if not self.cookies_check(self.cookies_names):
self.log.warning("no 'a' and 'b' session cookies set") self.log.warning("no 'a' and 'b' session cookies set")
FuraffinityExtractor._warning = False FuraffinityExtractor._warning = False
external = self.config("external", False) def items(self):
metadata = self.metadata() metadata = self.metadata()
for post_id in util.advance(self.posts(), self.offset): for post_id in util.advance(self.posts(), self.offset):
post = self._parse_post(post_id) post = self._parse_post(post_id)
@ -57,7 +57,7 @@ class FuraffinityExtractor(Extractor):
yield Message.Directory, post yield Message.Directory, post
yield Message.Url, post["url"], post yield Message.Url, post["url"], post
if external: if self.external:
for url in text.extract_iter( for url in text.extract_iter(
post["_description"], 'href="http', '"'): post["_description"], 'href="http', '"'):
yield Message.Queue, "http" + url, post yield Message.Queue, "http" + url, post
@ -219,12 +219,7 @@ class FuraffinityGalleryExtractor(FuraffinityExtractor):
"""Extractor for a furaffinity user's gallery""" """Extractor for a furaffinity user's gallery"""
subcategory = "gallery" subcategory = "gallery"
pattern = BASE_PATTERN + r"/gallery/([^/?#]+)" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)"
test = ("https://www.furaffinity.net/gallery/mirlinthloth/", { example = "https://www.furaffinity.net/gallery/USER/"
"pattern": r"https://d\d?\.f(uraffinity|acdn)\.net"
r"/art/mirlinthloth/\d+/\d+.\w+\.\w+",
"range": "45-50",
"count": 6,
})
def posts(self): def posts(self):
return self._pagination("gallery") return self._pagination("gallery")
@ -235,11 +230,7 @@ class FuraffinityScrapsExtractor(FuraffinityExtractor):
subcategory = "scraps" subcategory = "scraps"
directory_fmt = ("{category}", "{user!l}", "Scraps") directory_fmt = ("{category}", "{user!l}", "Scraps")
pattern = BASE_PATTERN + r"/scraps/([^/?#]+)" pattern = BASE_PATTERN + r"/scraps/([^/?#]+)"
test = ("https://www.furaffinity.net/scraps/mirlinthloth/", { example = "https://www.furaffinity.net/scraps/USER/"
"pattern": r"https://d\d?\.f(uraffinity|acdn)\.net"
r"/art/[^/]+(/stories)?/\d+/\d+.\w+.",
"count": ">= 3",
})
def posts(self): def posts(self):
return self._pagination("scraps") return self._pagination("scraps")
@ -250,13 +241,7 @@ class FuraffinityFavoriteExtractor(FuraffinityExtractor):
subcategory = "favorite" subcategory = "favorite"
directory_fmt = ("{category}", "{user!l}", "Favorites") directory_fmt = ("{category}", "{user!l}", "Favorites")
pattern = BASE_PATTERN + r"/favorites/([^/?#]+)" pattern = BASE_PATTERN + r"/favorites/([^/?#]+)"
test = ("https://www.furaffinity.net/favorites/mirlinthloth/", { example = "https://www.furaffinity.net/favorites/USER/"
"pattern": r"https://d\d?\.f(uraffinity|acdn)\.net"
r"/art/[^/]+/\d+/\d+.\w+\.\w+",
"keyword": {"favorite_id": int},
"range": "45-50",
"count": 6,
})
def posts(self): def posts(self):
return self._pagination_favorites() return self._pagination_favorites()
@ -273,19 +258,7 @@ class FuraffinitySearchExtractor(FuraffinityExtractor):
subcategory = "search" subcategory = "search"
directory_fmt = ("{category}", "Search", "{search}") directory_fmt = ("{category}", "Search", "{search}")
pattern = BASE_PATTERN + r"/search(?:/([^/?#]+))?/?[?&]([^#]+)" pattern = BASE_PATTERN + r"/search(?:/([^/?#]+))?/?[?&]([^#]+)"
test = ( example = "https://www.furaffinity.net/search/?q=QUERY"
("https://www.furaffinity.net/search/?q=cute", {
"pattern": r"https://d\d?\.f(uraffinity|acdn)\.net"
r"/art/[^/]+/\d+/\d+.\w+\.\w+",
"range": "45-50",
"count": 6,
}),
# first page of search results (#2402)
("https://www.furaffinity.net/search/?q=leaf&range=1day", {
"range": "1-3",
"count": 3,
}),
)
def __init__(self, match): def __init__(self, match):
FuraffinityExtractor.__init__(self, match) FuraffinityExtractor.__init__(self, match)
@ -304,65 +277,7 @@ class FuraffinityPostExtractor(FuraffinityExtractor):
"""Extractor for individual posts on furaffinity""" """Extractor for individual posts on furaffinity"""
subcategory = "post" subcategory = "post"
pattern = BASE_PATTERN + r"/(?:view|full)/(\d+)" pattern = BASE_PATTERN + r"/(?:view|full)/(\d+)"
test = ( example = "https://www.furaffinity.net/view/12345/"
("https://www.furaffinity.net/view/21835115/", {
"pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art"
r"/mirlinthloth/music/1488278723/1480267446.mirlinthlot"
r"h_dj_fennmink_-_bude_s_4_ever\.mp3",
"keyword": {
"artist" : "mirlinthloth",
"artist_url" : "mirlinthloth",
"date" : "dt:2016-11-27 17:24:06",
"description": "A Song made playing the game Cosmic DJ.",
"extension" : "mp3",
"filename" : r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever",
"id" : 21835115,
"tags" : list,
"title" : "Bude's 4 Ever",
"url" : r"re:https://d\d?\.f(uraffinity|acdn)\.net/art",
"user" : "mirlinthloth",
"views" : int,
"favorites" : int,
"comments" : int,
"rating" : "General",
"fa_category": "Music",
"theme" : "All",
"species" : "Unspecified / Any",
"gender" : "Any",
"width" : 120,
"height" : 120,
},
}),
# 'external' option (#1492)
("https://www.furaffinity.net/view/42166511/", {
"options": (("external", True),),
"pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/"
r"|http://www\.postybirb\.com",
"count": 2,
}),
# no tags (#2277)
("https://www.furaffinity.net/view/45331225/", {
"keyword": {
"artist": "Kota_Remminders",
"artist_url": "kotaremminders",
"date": "dt:2022-01-03 17:49:33",
"fa_category": "Adoptables",
"filename": "1641232173.kotaremminders_chidopts1",
"gender": "Any",
"height": 905,
"id": 45331225,
"rating": "General",
"species": "Unspecified / Any",
"tags": [],
"theme": "All",
"title": "REMINDER",
"width": 1280,
},
}),
("https://furaffinity.net/view/21835115/"),
("https://sfw.furaffinity.net/view/21835115/"),
("https://www.furaffinity.net/full/21835115/"),
)
def posts(self): def posts(self):
post_id = self.user post_id = self.user
@ -375,16 +290,7 @@ class FuraffinityUserExtractor(FuraffinityExtractor):
subcategory = "user" subcategory = "user"
cookies_domain = None cookies_domain = None
pattern = BASE_PATTERN + r"/user/([^/?#]+)" pattern = BASE_PATTERN + r"/user/([^/?#]+)"
test = ( example = "https://www.furaffinity.net/user/USER/"
("https://www.furaffinity.net/user/mirlinthloth/", {
"pattern": r"/gallery/mirlinthloth/$",
}),
("https://www.furaffinity.net/user/mirlinthloth/", {
"options": (("include", "all"),),
"pattern": r"/(gallery|scraps|favorites)/mirlinthloth/$",
"count": 3,
}),
)
def initialize(self): def initialize(self):
pass pass
@ -402,11 +308,7 @@ class FuraffinityFollowingExtractor(FuraffinityExtractor):
"""Extractor for a furaffinity user's watched users""" """Extractor for a furaffinity user's watched users"""
subcategory = "following" subcategory = "following"
pattern = BASE_PATTERN + "/watchlist/by/([^/?#]+)" pattern = BASE_PATTERN + "/watchlist/by/([^/?#]+)"
test = ("https://www.furaffinity.net/watchlist/by/mirlinthloth/", { example = "https://www.furaffinity.net/watchlist/by/USER/"
"pattern": FuraffinityUserExtractor.pattern,
"range": "176-225",
"count": 50,
})
def items(self): def items(self):
url = "{}/watchlist/by/{}/".format(self.root, self.user) url = "{}/watchlist/by/{}/".format(self.root, self.user)

27
gallery_dl/extractor/fuskator.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2019 Mike Fährmann # Copyright 2019-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -18,22 +18,7 @@ class FuskatorGalleryExtractor(GalleryExtractor):
category = "fuskator" category = "fuskator"
root = "https://fuskator.com" root = "https://fuskator.com"
pattern = r"(?:https?://)?fuskator\.com/(?:thumbs|expanded)/([^/?#]+)" pattern = r"(?:https?://)?fuskator\.com/(?:thumbs|expanded)/([^/?#]+)"
test = ( example = "https://fuskator.com/thumbs/ID/"
("https://fuskator.com/thumbs/d0GnIzXrSKU/", {
"pattern": r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg",
"count": 22,
"keyword": {
"gallery_id": 473023,
"gallery_hash": "d0GnIzXrSKU",
"title": "re:Shaved Brunette Babe Maria Ryabushkina with ",
"views": int,
"score": float,
"count": 22,
"tags": list,
},
}),
("https://fuskator.com/expanded/gXpKzjgIidA/index.html"),
)
def __init__(self, match): def __init__(self, match):
self.gallery_hash = match.group(1) self.gallery_hash = match.group(1)
@ -82,13 +67,7 @@ class FuskatorSearchExtractor(Extractor):
subcategory = "search" subcategory = "search"
root = "https://fuskator.com" root = "https://fuskator.com"
pattern = r"(?:https?://)?fuskator\.com(/(?:search|page)/.+)" pattern = r"(?:https?://)?fuskator\.com(/(?:search|page)/.+)"
test = ( example = "https://fuskator.com/search/TAG/"
("https://fuskator.com/search/red_swimsuit/", {
"pattern": FuskatorGalleryExtractor.pattern,
"count": ">= 40",
}),
("https://fuskator.com/page/3/swimsuit/quality/"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

83
gallery_dl/extractor/gelbooru.py
View File

@ -115,18 +115,7 @@ class GelbooruTagExtractor(GelbooruBase,
gelbooru_v02.GelbooruV02TagExtractor): gelbooru_v02.GelbooruV02TagExtractor):
"""Extractor for images from gelbooru.com based on search-tags""" """Extractor for images from gelbooru.com based on search-tags"""
pattern = BASE_PATTERN + r"page=post&s=list&tags=([^&#]+)" pattern = BASE_PATTERN + r"page=post&s=list&tags=([^&#]+)"
test = ( example = "https://gelbooru.com/index.php?page=post&s=list&tags=TAG"
("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", {
"count": 5,
}),
("https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", {
"range": "196-204",
"url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf",
"pattern": r"https://img\d\.gelbooru\.com"
r"/images/../../[0-9a-f]{32}\.jpg",
"count": 9,
}),
)
class GelbooruPoolExtractor(GelbooruBase, class GelbooruPoolExtractor(GelbooruBase,
@ -134,11 +123,7 @@ class GelbooruPoolExtractor(GelbooruBase,
"""Extractor for gelbooru pools""" """Extractor for gelbooru pools"""
per_page = 45 per_page = 45
pattern = BASE_PATTERN + r"page=pool&s=show&id=(\d+)" pattern = BASE_PATTERN + r"page=pool&s=show&id=(\d+)"
test = ( example = "https://gelbooru.com/index.php?page=pool&s=show&id=12345"
("https://gelbooru.com/index.php?page=pool&s=show&id=761", {
"count": 6,
}),
)
skip = GelbooruBase._skip_offset skip = GelbooruBase._skip_offset
@ -169,9 +154,7 @@ class GelbooruFavoriteExtractor(GelbooruBase,
"""Extractor for gelbooru favorites""" """Extractor for gelbooru favorites"""
per_page = 100 per_page = 100
pattern = BASE_PATTERN + r"page=favorites&s=view&id=(\d+)" pattern = BASE_PATTERN + r"page=favorites&s=view&id=(\d+)"
test = ("https://gelbooru.com/index.php?page=favorites&s=view&id=279415", { example = "https://gelbooru.com/index.php?page=favorites&s=view&id=12345"
"count": 3,
})
skip = GelbooruBase._skip_offset skip = GelbooruBase._skip_offset
@ -221,70 +204,14 @@ class GelbooruPostExtractor(GelbooruBase,
r"(?=(?:[^#]+&)?page=post(?:&|#|$))" r"(?=(?:[^#]+&)?page=post(?:&|#|$))"
r"(?=(?:[^#]+&)?s=view(?:&|#|$))" r"(?=(?:[^#]+&)?s=view(?:&|#|$))"
r"(?:[^#]+&)?id=(\d+)") r"(?:[^#]+&)?id=(\d+)")
test = ( example = "https://gelbooru.com/index.php?page=post&s=view&id=12345"
("https://gelbooru.com/index.php?page=post&s=view&id=313638", {
"content": "5e255713cbf0a8e0801dc423563c34d896bb9229",
"count": 1,
}),
("https://gelbooru.com/index.php?page=post&s=view&id=313638"),
("https://gelbooru.com/index.php?s=view&page=post&id=313638"),
("https://gelbooru.com/index.php?page=post&id=313638&s=view"),
("https://gelbooru.com/index.php?s=view&id=313638&page=post"),
("https://gelbooru.com/index.php?id=313638&page=post&s=view"),
("https://gelbooru.com/index.php?id=313638&s=view&page=post"),
("https://gelbooru.com/index.php?page=post&s=view&id=6018318", {
"options": (("tags", True),),
"content": "977caf22f27c72a5d07ea4d4d9719acdab810991",
"keyword": {
"tags_artist": "kirisaki_shuusei",
"tags_character": str,
"tags_copyright": "vocaloid",
"tags_general": str,
"tags_metadata": str,
},
}),
# video
("https://gelbooru.com/index.php?page=post&s=view&id=5938076", {
"content": "6360452fa8c2f0c1137749e81471238564df832a",
"pattern": r"https://img\d\.gelbooru\.com/images"
r"/22/61/226111273615049235b001b381707bd0\.webm",
}),
# notes
("https://gelbooru.com/index.php?page=post&s=view&id=5997331", {
"options": (("notes", True),),
"keyword": {
"notes": [
{
"body": "Look over this way when you talk~",
"height": 553,
"width": 246,
"x": 35,
"y": 72,
},
{
"body": "Hey~\nAre you listening~?",
"height": 557,
"width": 246,
"x": 1233,
"y": 109,
},
],
},
}),
)
class GelbooruRedirectExtractor(GelbooruBase, Extractor): class GelbooruRedirectExtractor(GelbooruBase, Extractor):
subcategory = "redirect" subcategory = "redirect"
pattern = (r"(?:https?://)?(?:www\.)?gelbooru\.com" pattern = (r"(?:https?://)?(?:www\.)?gelbooru\.com"
r"/redirect\.php\?s=([^&#]+)") r"/redirect\.php\?s=([^&#]+)")
test = (("https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgu" example = "https://gelbooru.com/redirect.php?s=BASE64"
"cGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng=="), {
"pattern": r"https://gelbooru.com/index.php"
r"\?page=post&s=view&id=1830046"
})
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

70
gallery_dl/extractor/gelbooru_v01.py
View File

@ -90,24 +90,7 @@ class GelbooruV01TagExtractor(GelbooruV01Extractor):
directory_fmt = ("{category}", "{search_tags}") directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}" archive_fmt = "t_{search_tags}_{id}"
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)"
test = ( example = "https://allgirl.booru.org/index.php?page=post&s=list&tags=TAG"
(("https://the-collection.booru.org"
"/index.php?page=post&s=list&tags=parody"), {
"range": "1-25",
"count": 25,
}),
(("https://illusioncards.booru.org"
"/index.php?page=post&s=list&tags=koikatsu"), {
"range": "1-25",
"count": 25,
}),
("https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", {
"range": "1-25",
"count": 25,
}),
("https://drawfriends.booru.org/index.php?page=post&s=list&tags=all"),
("https://vidyart2.booru.org/index.php?page=post&s=list&tags=all"),
)
def __init__(self, match): def __init__(self, match):
GelbooruV01Extractor.__init__(self, match) GelbooruV01Extractor.__init__(self, match)
@ -128,21 +111,7 @@ class GelbooruV01FavoriteExtractor(GelbooruV01Extractor):
archive_fmt = "f_{favorite_id}_{id}" archive_fmt = "f_{favorite_id}_{id}"
per_page = 50 per_page = 50
pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)" pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)"
test = ( example = "https://allgirl.booru.org/index.php?page=favorites&s=view&id=N"
(("https://the-collection.booru.org"
"/index.php?page=favorites&s=view&id=1166"), {
"count": 2,
}),
(("https://illusioncards.booru.org"
"/index.php?page=favorites&s=view&id=84887"), {
"count": 2,
}),
("https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", {
"count": 4,
}),
("https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1"),
("https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1"),
)
def __init__(self, match): def __init__(self, match):
GelbooruV01Extractor.__init__(self, match) GelbooruV01Extractor.__init__(self, match)
@ -161,40 +130,7 @@ class GelbooruV01PostExtractor(GelbooruV01Extractor):
subcategory = "post" subcategory = "post"
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)"
test = ( example = "https://allgirl.booru.org/index.php?page=post&s=view&id=12345"
(("https://the-collection.booru.org"
"/index.php?page=post&s=view&id=100520"), {
"url": "0329ac8588bb93cf242ca0edbe3e995b4ba554e8",
"content": "1e585874e7b874f7937df1060dd1517fef2f4dfb",
}),
(("https://illusioncards.booru.org"
"/index.php?page=post&s=view&id=82746"), {
"url": "3f9cd2fadf78869b90bc5422f27b48f1af0e0909",
"content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc",
}),
("https://allgirl.booru.org/index.php?page=post&s=view&id=107213", {
"url": "b416800d2d2b072f80d3b37cfca9cb806fb25d51",
"content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7",
"keyword": {
"created_at": "2021-02-13 16:27:39",
"date": "dt:2021-02-13 16:27:39",
"file_url": "https://img.booru.org/allgirl//images/107"
"/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg",
"height": "1200",
"id": "107213",
"md5": "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb",
"rating": "s",
"score": str,
"source": "",
"tags": "blush dress green_eyes green_hair hatsune_miku "
"long_hair twintails vocaloid",
"uploader": "Honochi31",
"width": "1600"
},
}),
("https://drawfriends.booru.org/index.php?page=post&s=view&id=107474"),
("https://vidyart2.booru.org/index.php?page=post&s=view&id=39168"),
)
def __init__(self, match): def __init__(self, match):
GelbooruV01Extractor.__init__(self, match) GelbooruV01Extractor.__init__(self, match)

178
gallery_dl/extractor/gelbooru_v02.py
View File

@ -197,30 +197,7 @@ class GelbooruV02TagExtractor(GelbooruV02Extractor):
directory_fmt = ("{category}", "{search_tags}") directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}" archive_fmt = "t_{search_tags}_{id}"
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=list&tags=([^&#]+)"
test = ( example = "https://safebooru.org/index.php?page=post&s=list&tags=TAG"
("https://rule34.xxx/index.php?page=post&s=list&tags=danraku", {
"content": ("5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c",
"622e80be3f496672c44aab5c47fbc6941c61bc79"),
"pattern": r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg",
"count": 2,
}),
("https://safebooru.org/index.php?page=post&s=list&tags=bonocho", {
"url": "17c61b386530cf4c30842c9f580d15ef1cd09586",
"content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb",
}),
("https://realbooru.com/index.php?page=post&s=list&tags=wine", {
"count": ">= 64",
}),
("https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", {
"count": ">= 120",
}),
("https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", {
"url": "fe662b86d38c331fcac9c62af100167d404937dc",
}),
("https://xbooru.com/index.php?page=post&s=list&tags=konoyan", {
"count": 11,
}),
)
def __init__(self, match): def __init__(self, match):
GelbooruV02Extractor.__init__(self, match) GelbooruV02Extractor.__init__(self, match)
@ -239,25 +216,7 @@ class GelbooruV02PoolExtractor(GelbooruV02Extractor):
directory_fmt = ("{category}", "pool", "{pool}") directory_fmt = ("{category}", "pool", "{pool}")
archive_fmt = "p_{pool}_{id}" archive_fmt = "p_{pool}_{id}"
pattern = BASE_PATTERN + r"/index\.php\?page=pool&s=show&id=(\d+)" pattern = BASE_PATTERN + r"/index\.php\?page=pool&s=show&id=(\d+)"
test = ( example = "https://safebooru.org/index.php?page=pool&s=show&id=12345"
("https://rule34.xxx/index.php?page=pool&s=show&id=179", {
"count": 3,
}),
("https://safebooru.org/index.php?page=pool&s=show&id=11", {
"count": 5,
}),
("https://realbooru.com/index.php?page=pool&s=show&id=1", {
"count": 3,
}),
("https://hypnohub.net/index.php?page=pool&s=show&id=61", {
"url": "d314826280073441a2da609f70ee814d1f4b9407",
"count": 3,
}),
("https://xbooru.com/index.php?page=pool&s=show&id=757", {
"url": "ceeac56c179ec72301bd0b6add6355a138fdea01",
"count": 5,
}),
)
def __init__(self, match): def __init__(self, match):
GelbooruV02Extractor.__init__(self, match) GelbooruV02Extractor.__init__(self, match)
@ -309,26 +268,7 @@ class GelbooruV02FavoriteExtractor(GelbooruV02Extractor):
archive_fmt = "f_{favorite_id}_{id}" archive_fmt = "f_{favorite_id}_{id}"
per_page = 50 per_page = 50
pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)" pattern = BASE_PATTERN + r"/index\.php\?page=favorites&s=view&id=(\d+)"
test = ( example = "https://safebooru.org/index.php?page=favorites&s=view&id=12345"
("https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", {
"count": 3,
}),
("https://safebooru.org/index.php?page=favorites&s=view&id=17567", {
"count": 2,
}),
("https://realbooru.com/index.php?page=favorites&s=view&id=274", {
"count": 2,
}),
("https://tbib.org/index.php?page=favorites&s=view&id=7881", {
"count": 3,
}),
("https://hypnohub.net/index.php?page=favorites&s=view&id=43546", {
"count": 3,
}),
("https://xbooru.com/index.php?page=favorites&s=view&id=45206", {
"count": 4,
}),
)
def __init__(self, match): def __init__(self, match):
GelbooruV02Extractor.__init__(self, match) GelbooruV02Extractor.__init__(self, match)
@ -349,117 +289,7 @@ class GelbooruV02PostExtractor(GelbooruV02Extractor):
subcategory = "post" subcategory = "post"
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)" pattern = BASE_PATTERN + r"/index\.php\?page=post&s=view&id=(\d+)"
test = ( example = "https://safebooru.org/index.php?page=post&s=view&id=12345"
("https://rule34.xxx/index.php?page=post&s=view&id=863", {
"pattern": r"https://api-cdn\.rule34\.xxx/images"
r"/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg",
"content": ("a43f418aa350039af0d11cae501396a33bbe2201",
"67b516295950867e1c1ab6bc13b35d3b762ed2a3"),
"options": (("tags", True), ("notes", True)),
"keyword": {
"tags_artist": "reverse_noise yamu_(reverse_noise)",
"tags_character": "hong_meiling",
"tags_copyright": "touhou",
"tags_general": str,
"tags_metadata": "censored translated",
"notes": [
{
"body": "It feels angry, I'm losing myself... "
"It won't calm down!",
"height": 65,
"id": 93586,
"width": 116,
"x": 22,
"y": 333,
},
{
"body": "REPUTATION OF RAGE",
"height": 272,
"id": 93587,
"width": 199,
"x": 78,
"y": 442,
},
],
},
}),
("https://hypnohub.net/index.php?page=post&s=view&id=1439", {
"pattern": r"https://hypnohub\.net/images"
r"/90/24/90245c3c5250c2a8173255d3923a010b\.jpg",
"content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71",
"options": (("tags", True), ("notes", True)),
"keyword": {
"tags_artist": "brokenteapot",
"tags_character": "hsien-ko",
"tags_copyright": "capcom darkstalkers",
"tags_general": str,
"tags_metadata": "dialogue text translated",
"notes": [
{
"body": "Master Master Master "
"Master Master Master",
"height": 83,
"id": 10577,
"width": 129,
"x": 259,
"y": 20,
},
{
"body": "Response Response Response "
"Response Response Response",
"height": 86,
"id": 10578,
"width": 125,
"x": 126,
"y": 20,
},
{
"body": "Obedience Obedience Obedience "
"Obedience Obedience Obedience",
"height": 80,
"id": 10579,
"width": 98,
"x": 20,
"y": 20,
},
],
},
}),
("https://safebooru.org/index.php?page=post&s=view&id=1169132", {
"url": "cf05e37a3c62b2d55788e2080b8eabedb00f999b",
"content": "93b293b27dabd198afafabbaf87c49863ac82f27",
"options": (("tags", True),),
"keyword": {
"tags_artist": "kawanakajima",
"tags_character": "heath_ledger ronald_mcdonald the_joker",
"tags_copyright": "dc_comics mcdonald's the_dark_knight",
"tags_general": str,
},
}),
("https://realbooru.com/index.php?page=post&s=view&id=668483", {
"pattern": r"https://realbooru\.com//?images/dc/b5"
r"/dcb5c0ce9ec0bf74a6930608985f4719\.jpeg",
"content": "7f5873ce3b6cd295ea2e81fcb49583098ea9c8da",
"options": (("tags", True),),
"keyword": {
"tags_general": "1girl blonde blonde_hair blue_eyes cute "
"female female_only looking_at_viewer smile "
"solo solo_female teeth",
"tags_model": "jennifer_lawrence",
},
}),
("https://tbib.org/index.php?page=post&s=view&id=9233957", {
"url": "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2",
"content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43",
}),
("https://xbooru.com/index.php?page=post&s=view&id=1025649", {
"pattern": r"https://img\.xbooru\.com/images/444"
r"/f3eda549ad8b9db244ac335c7406c92f\.jpeg",
"content": "086668afd445438d491ecc11cee3ac69b4d65530",
}),
)
def __init__(self, match): def __init__(self, match):
GelbooruV02Extractor.__init__(self, match) GelbooruV02Extractor.__init__(self, match)

26
gallery_dl/extractor/generic.py
View File

@ -34,31 +34,7 @@ class GenericExtractor(Extractor):
r"(?:\?(?P<query>[^#]*))?" # optional query r"(?:\?(?P<query>[^#]*))?" # optional query
r"(?:\#(?P<fragment>.*))?" # optional fragment r"(?:\#(?P<fragment>.*))?" # optional fragment
) )
example = "generic:https://www.nongnu.org/lzip/"
test = (
("generic:https://www.nongnu.org/lzip/", {
"count": 1,
"content": "40be5c77773d3e91db6e1c5df720ee30afb62368",
"keyword": {
"description": "Lossless data compressor",
"imageurl": "https://www.nongnu.org/lzip/lzip.png",
"keywords": "lzip, clzip, plzip, lzlib, LZMA, bzip2, "
"gzip, data compression, GNU, free software",
"pageurl": "https://www.nongnu.org/lzip/",
},
}),
# internationalized domain name
("generic:https://räksmörgås.josefsson.org/", {
"count": 2,
"pattern": "^https://räksmörgås.josefsson.org/",
}),
("g:https://en.wikipedia.org/Main_Page"),
("g:https://example.org/path/to/file?que=1?&ry=2/#fragment"),
("g:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E"),
("generic:https://en.wikipedia.org/Main_Page"),
("generic:https://example.org/path/to/file?que=1?&ry=2/#fragment"),
("generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

46
gallery_dl/extractor/gofile.py
View File

@ -4,6 +4,8 @@
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation. # published by the Free Software Foundation.
"""Extractors for https://gofile.io/"""
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, exception from .. import text, exception
from ..cache import cache, memcache from ..cache import cache, memcache
@ -17,49 +19,7 @@ class GofileFolderExtractor(Extractor):
directory_fmt = ("{category}", "{name} ({code})") directory_fmt = ("{category}", "{name} ({code})")
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = r"(?:https?://)?(?:www\.)?gofile\.io/d/([^/?#]+)" pattern = r"(?:https?://)?(?:www\.)?gofile\.io/d/([^/?#]+)"
test = ( example = "https://gofile.io/d/ID"
("https://gofile.io/d/k6BomI", {
"pattern": r"https://store\d+\.gofile\.io/download"
r"/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}"
r"/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png",
"keyword": {
"createTime": int,
"directLink": "re:https://store5.gofile.io/download/direct/.+",
"downloadCount": int,
"extension": "png",
"filename": "test-テスト-%22&!",
"folder": {
"childs": [
"b0367d79-b8ba-407f-8342-aaf8eb815443",
"7fd4a36a-c1dd-49ff-9223-d93f7d24093f"
],
"code": "k6BomI",
"createTime": 1654076165,
"id": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c",
"name": "root",
"public": True,
"totalDownloadCount": int,
"totalSize": 182,
"type": "folder"
},
"id": r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}",
"link": r"re:https://store5.gofile.io/download/.+\.png",
"md5": "re:[0-9a-f]{32}",
"mimetype": "image/png",
"name": "test-テスト-%22&!.png",
"num": int,
"parentFolder": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c",
"serverChoosen": "store5",
"size": 182,
"thumbnail": r"re:https://store5.gofile.io/download/.+\.png",
"type": "file"
},
}),
("https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", {
"options": (("website-token", None),),
"content": "0c8768055e4e20e7c7259608b67799171b691140",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

11
gallery_dl/extractor/hbrowse.py
View File

@ -47,11 +47,7 @@ class HbrowseChapterExtractor(HbrowseBase, ChapterExtractor):
"{page:>03}.{extension}") "{page:>03}.{extension}")
archive_fmt = "{manga_id}_{chapter}_{page}" archive_fmt = "{manga_id}_{chapter}_{page}"
pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/(\d+)/c(\d+))" pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/(\d+)/c(\d+))"
test = ("https://www.hbrowse.com/10363/c00000", { example = "https://www.hbrowse.com/12345/c00000"
"url": "6feefbc9f4b98e20d8425ddffa9dd111791dc3e6",
"keyword": "274996f6c809e5250b6ff3abbc5147e29f89d9a5",
"content": "44578ebbe176c2c27434966aef22945787e2781e",
})
def __init__(self, match): def __init__(self, match):
self.path, self.gid, self.chapter = match.groups() self.path, self.gid, self.chapter = match.groups()
@ -75,10 +71,7 @@ class HbrowseMangaExtractor(HbrowseBase, MangaExtractor):
chapterclass = HbrowseChapterExtractor chapterclass = HbrowseChapterExtractor
reverse = False reverse = False
pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/\d+)/?$" pattern = r"(?:https?://)?(?:www\.)?hbrowse\.com(/\d+)/?$"
test = ("https://www.hbrowse.com/10363", { example = "https://www.hbrowse.com/12345"
"url": "b89682bfb86c11d2af0dc47463804ec3ac4aadd6",
"keyword": "4b15fda1858a69de1fbf5afddfe47dd893397312",
})
def chapters(self, page): def chapters(self, page):
results = [] results = []

54
gallery_dl/extractor/hentai2read.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2016-2022 Mike Fährmann # Copyright 2016-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -23,31 +23,7 @@ class Hentai2readChapterExtractor(Hentai2readBase, ChapterExtractor):
"""Extractor for a single manga chapter from hentai2read.com""" """Extractor for a single manga chapter from hentai2read.com"""
archive_fmt = "{chapter_id}_{page}" archive_fmt = "{chapter_id}_{page}"
pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+/([^/?#]+))" pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+/([^/?#]+))"
test = ( example = "https://hentai2read.com/TITLE/1/"
("https://hentai2read.com/amazon_elixir/1/", {
"url": "964b942cf492b3a129d2fe2608abfc475bc99e71",
"keyword": "85645b02d34aa11b3deb6dadd7536863476e1bad",
}),
("https://hentai2read.com/popuni_kei_joshi_panic/2.5/", {
"pattern": r"https://hentaicdn\.com/hentai"
r"/13088/2\.5y/ccdn00\d+\.jpg",
"count": 36,
"keyword": {
"author": "Kurisu",
"chapter": 2,
"chapter_id": 75152,
"chapter_minor": ".5",
"count": 36,
"lang": "en",
"language": "English",
"manga": "Popuni Kei Joshi Panic!",
"manga_id": 13088,
"page": int,
"title": "Popuni Kei Joshi Panic! 2.5",
"type": "Original",
},
}),
)
def __init__(self, match): def __init__(self, match):
self.chapter = match.group(2) self.chapter = match.group(2)
@ -85,31 +61,7 @@ class Hentai2readMangaExtractor(Hentai2readBase, MangaExtractor):
"""Extractor for hmanga from hentai2read.com""" """Extractor for hmanga from hentai2read.com"""
chapterclass = Hentai2readChapterExtractor chapterclass = Hentai2readChapterExtractor
pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+)/?$" pattern = r"(?:https?://)?(?:www\.)?hentai2read\.com(/[^/?#]+)/?$"
test = ( example = "https://hentai2read.com/TITLE/"
("https://hentai2read.com/amazon_elixir/", {
"url": "273073752d418ec887d7f7211e42b832e8c403ba",
"keyword": "5c1b712258e78e120907121d3987c71f834d13e1",
}),
("https://hentai2read.com/oshikage_riot/", {
"url": "6595f920a3088a15c2819c502862d45f8eb6bea6",
"keyword": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af",
}),
("https://hentai2read.com/popuni_kei_joshi_panic/", {
"pattern": Hentai2readChapterExtractor.pattern,
"range": "2-3",
"keyword": {
"chapter": int,
"chapter_id": int,
"chapter_minor": ".5",
"lang": "en",
"language": "English",
"manga": "Popuni Kei Joshi Panic!",
"manga_id": 13088,
"title": str,
"type": "Original",
},
}),
)
def chapters(self, page): def chapters(self, page):
results = [] results = []

31
gallery_dl/extractor/hentaicosplays.py
View File

@ -21,36 +21,7 @@ class HentaicosplaysGalleryExtractor(GalleryExtractor):
pattern = r"((?:https?://)?(?:\w{2}\.)?" \ pattern = r"((?:https?://)?(?:\w{2}\.)?" \
r"(hentai-cosplays|hentai-img|porn-images-xxx)\.com)/" \ r"(hentai-cosplays|hentai-img|porn-images-xxx)\.com)/" \
r"(?:image|story)/([\w-]+)" r"(?:image|story)/([\w-]+)"
test = ( example = "https://hentai-cosplays.com/image/TITLE/"
("https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", {
"pattern": r"https://static\d?.hentai-cosplays.com/upload/"
r"\d+/\d+/\d+/\d+.jpg$",
"keyword": {
"count": 18,
"site": "hentai-cosplays",
"slug": "---devilism--tide-kurihara-",
"title": "艦 こ れ-devilism の tide Kurihara 憂",
},
}),
("https://fr.porn-images-xxx.com/image/enako-enako-24/", {
"pattern": r"https://static\d?.porn-images-xxx.com/upload/"
r"\d+/\d+/\d+/\d+.jpg$",
"keyword": {
"count": 11,
"site": "porn-images-xxx",
"title": str,
},
}),
("https://ja.hentai-img.com/image/hollow-cora-502/", {
"pattern": r"https://static\d?.hentai-img.com/upload/"
r"\d+/\d+/\d+/\d+.jpg$",
"keyword": {
"count": 2,
"site": "hentai-img",
"title": str,
},
}),
)
def __init__(self, match): def __init__(self, match):
root, self.site, self.slug = match.groups() root, self.site, self.slug = match.groups()

91
gallery_dl/extractor/hentaifoundry.py
View File

@ -168,7 +168,7 @@ class HentaifoundryUserExtractor(HentaifoundryExtractor):
"""Extractor for a hentaifoundry user profile""" """Extractor for a hentaifoundry user profile"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/user/([^/?#]+)/profile" pattern = BASE_PATTERN + r"/user/([^/?#]+)/profile"
test = ("https://www.hentai-foundry.com/user/Tenpura/profile",) example = "https://www.hentai-foundry.com/user/USER/profile"
def initialize(self): def initialize(self):
pass pass
@ -192,12 +192,7 @@ class HentaifoundryPicturesExtractor(HentaifoundryExtractor):
"""Extractor for all pictures of a hentaifoundry user""" """Extractor for all pictures of a hentaifoundry user"""
subcategory = "pictures" subcategory = "pictures"
pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)(?:/page/(\d+))?/?$" pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)(?:/page/(\d+))?/?$"
test = ( example = "https://www.hentai-foundry.com/pictures/user/USER"
("https://www.hentai-foundry.com/pictures/user/Tenpura", {
"url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28",
}),
("https://www.hentai-foundry.com/pictures/user/Tenpura/page/3"),
)
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match) HentaifoundryExtractor.__init__(self, match)
@ -209,13 +204,7 @@ class HentaifoundryScrapsExtractor(HentaifoundryExtractor):
subcategory = "scraps" subcategory = "scraps"
directory_fmt = ("{category}", "{user}", "Scraps") directory_fmt = ("{category}", "{user}", "Scraps")
pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)/scraps" pattern = BASE_PATTERN + r"/pictures/user/([^/?#]+)/scraps"
test = ( example = "https://www.hentai-foundry.com/pictures/user/USER/scraps"
("https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps", {
"url": "7cd9c6ec6258c4ab8c44991f7731be82337492a7",
}),
("https://www.hentai-foundry.com"
"/pictures/user/Evulchibi/scraps/page/3"),
)
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match) HentaifoundryExtractor.__init__(self, match)
@ -229,13 +218,7 @@ class HentaifoundryFavoriteExtractor(HentaifoundryExtractor):
directory_fmt = ("{category}", "{user}", "Favorites") directory_fmt = ("{category}", "{user}", "Favorites")
archive_fmt = "f_{user}_{index}" archive_fmt = "f_{user}_{index}"
pattern = BASE_PATTERN + r"/user/([^/?#]+)/faves/pictures" pattern = BASE_PATTERN + r"/user/([^/?#]+)/faves/pictures"
test = ( example = "https://www.hentai-foundry.com/user/USER/faves/pictures"
("https://www.hentai-foundry.com/user/Tenpura/faves/pictures", {
"url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b",
}),
("https://www.hentai-foundry.com"
"/user/Tenpura/faves/pictures/page/3"),
)
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match) HentaifoundryExtractor.__init__(self, match)
@ -249,10 +232,7 @@ class HentaifoundryRecentExtractor(HentaifoundryExtractor):
directory_fmt = ("{category}", "Recent Pictures", "{date}") directory_fmt = ("{category}", "Recent Pictures", "{date}")
archive_fmt = "r_{index}" archive_fmt = "r_{index}"
pattern = BASE_PATTERN + r"/pictures/recent/(\d\d\d\d-\d\d-\d\d)" pattern = BASE_PATTERN + r"/pictures/recent/(\d\d\d\d-\d\d-\d\d)"
test = ("https://www.hentai-foundry.com/pictures/recent/2018-09-20", { example = "https://www.hentai-foundry.com/pictures/recent/YYYY-MM-DD"
"pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/",
"range": "20-30",
})
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match) HentaifoundryExtractor.__init__(self, match)
@ -268,10 +248,7 @@ class HentaifoundryPopularExtractor(HentaifoundryExtractor):
directory_fmt = ("{category}", "Popular Pictures") directory_fmt = ("{category}", "Popular Pictures")
archive_fmt = "p_{index}" archive_fmt = "p_{index}"
pattern = BASE_PATTERN + r"/pictures/popular()" pattern = BASE_PATTERN + r"/pictures/popular()"
test = ("https://www.hentai-foundry.com/pictures/popular", { example = "https://www.hentai-foundry.com/pictures/popular"
"pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/",
"range": "20-30",
})
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match) HentaifoundryExtractor.__init__(self, match)
@ -283,34 +260,8 @@ class HentaifoundryImageExtractor(HentaifoundryExtractor):
subcategory = "image" subcategory = "image"
pattern = (r"(https?://)?(?:www\.|pictures\.)?hentai-foundry\.com" pattern = (r"(https?://)?(?:www\.|pictures\.)?hentai-foundry\.com"
r"/(?:pictures/user|[^/?#])/([^/?#]+)/(\d+)") r"/(?:pictures/user|[^/?#])/([^/?#]+)/(\d+)")
test = ( example = "https://www.hentai-foundry.com/pictures/user/USER/12345/TITLE"
(("https://www.hentai-foundry.com"
"/pictures/user/Tenpura/407501/shimakaze"), {
"url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3",
"content": "91bf01497c39254b6dfb234a18e8f01629c77fd1",
"keyword": {
"artist" : "Tenpura",
"date" : "dt:2016-02-22 14:41:19",
"description": "Thank you!",
"height" : 700,
"index" : 407501,
"media" : "Other digital art",
"ratings": ["Sexual content", "Contains female nudity"],
"score" : int,
"tags" : ["collection", "kancolle", "kantai", "shimakaze"],
"title" : "shimakaze",
"user" : "Tenpura",
"views" : int,
"width" : 495,
},
}),
("http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", {
"pattern": "http://pictures.hentai-foundry.com/t/Tenpura/407501/",
}),
("https://www.hentai-foundry.com/pictures/user/Tenpura/407501/"),
("https://pictures.hentai-foundry.com"
"/t/Tenpura/407501/Tenpura-407501-shimakaze.png"),
)
skip = Extractor.skip skip = Extractor.skip
def __init__(self, match): def __init__(self, match):
@ -331,24 +282,7 @@ class HentaifoundryStoriesExtractor(HentaifoundryExtractor):
subcategory = "stories" subcategory = "stories"
archive_fmt = "s_{index}" archive_fmt = "s_{index}"
pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)(?:/page/(\d+))?/?$" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)(?:/page/(\d+))?/?$"
test = ("https://www.hentai-foundry.com/stories/user/SnowWolf35", { example = "https://www.hentai-foundry.com/stories/user/USER"
"count": ">= 35",
"keyword": {
"author" : "SnowWolf35",
"chapters" : int,
"comments" : int,
"date" : "type:datetime",
"description": str,
"index" : int,
"rating" : int,
"ratings" : list,
"status" : "re:(Inc|C)omplete",
"title" : str,
"user" : "SnowWolf35",
"views" : int,
"words" : int,
},
})
def items(self): def items(self):
self._init_site_filters() self._init_site_filters()
@ -367,11 +301,8 @@ class HentaifoundryStoryExtractor(HentaifoundryExtractor):
subcategory = "story" subcategory = "story"
archive_fmt = "s_{index}" archive_fmt = "s_{index}"
pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)/(\d+)" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)/(\d+)"
test = (("https://www.hentai-foundry.com/stories/user/SnowWolf35" example = "https://www.hentai-foundry.com/stories/user/USER/12345/TITLE"
"/26416/Overwatch-High-Chapter-Voting-Location"), {
"url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8",
"keyword": {"title": "Overwatch High Chapter Voting Location"},
})
skip = Extractor.skip skip = Extractor.skip
def __init__(self, match): def __init__(self, match):

52
gallery_dl/extractor/hentaifox.py
View File

@ -21,40 +21,7 @@ class HentaifoxBase():
class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor): class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor):
"""Extractor for image galleries on hentaifox.com""" """Extractor for image galleries on hentaifox.com"""
pattern = r"(?:https?://)?(?:www\.)?hentaifox\.com(/gallery/(\d+))" pattern = r"(?:https?://)?(?:www\.)?hentaifox\.com(/gallery/(\d+))"
test = ( example = "https://hentaifox.com/gallery/12345/"
("https://hentaifox.com/gallery/56622/", {
"pattern": r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg",
"keyword": "bcd6b67284f378e5cc30b89b761140e3e60fcd92",
"count": 24,
}),
# 'split_tag' element (#1378)
("https://hentaifox.com/gallery/630/", {
"keyword": {
"artist": ["beti", "betty", "magi", "mimikaki"],
"characters": [
"aerith gainsborough",
"tifa lockhart",
"yuffie kisaragi"
],
"count": 32,
"gallery_id": 630,
"group": ["cu-little2"],
"parody": ["darkstalkers | vampire", "final fantasy vii"],
"tags": ["femdom", "fingering", "masturbation", "yuri"],
"title": "Cu-Little Bakanya～",
"type": "doujinshi",
},
}),
# email-protected title (#4201)
("https://hentaifox.com/gallery/35261/", {
"keyword": {
"gallery_id": 35261,
"title": "ManageM@ster!",
"artist": ["haritama hiroki"],
"group": ["studio n.ball"],
},
}),
)
def __init__(self, match): def __init__(self, match):
GalleryExtractor.__init__(self, match) GalleryExtractor.__init__(self, match)
@ -116,22 +83,7 @@ class HentaifoxSearchExtractor(HentaifoxBase, Extractor):
subcategory = "search" subcategory = "search"
pattern = (r"(?:https?://)?(?:www\.)?hentaifox\.com" pattern = (r"(?:https?://)?(?:www\.)?hentaifox\.com"
r"(/(?:parody|tag|artist|character|search|group)/[^/?%#]+)") r"(/(?:parody|tag|artist|character|search|group)/[^/?%#]+)")
test = ( example = "https://hentaifox.com/tag/TAG/"
("https://hentaifox.com/parody/touhou-project/"),
("https://hentaifox.com/character/reimu-hakurei/"),
("https://hentaifox.com/artist/distance/"),
("https://hentaifox.com/search/touhou/"),
("https://hentaifox.com/group/v-slash/"),
("https://hentaifox.com/tag/heterochromia/", {
"pattern": HentaifoxGalleryExtractor.pattern,
"count": ">= 60",
"keyword": {
"url" : str,
"gallery_id": int,
"title" : str,
},
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

32
gallery_dl/extractor/hentaihand.py
View File

@ -17,27 +17,7 @@ class HentaihandGalleryExtractor(GalleryExtractor):
category = "hentaihand" category = "hentaihand"
root = "https://hentaihand.com" root = "https://hentaihand.com"
pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)" pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)"
test = ( example = "https://hentaihand.com/en/comic/TITLE"
(("https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-"
"no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-"
"no-railgun-english"), {
"pattern": r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$",
"count": 50,
"keyword": {
"artists" : ["Takumi Na Muchi"],
"date" : "dt:2014-06-28 00:00:00",
"gallery_id": 37387,
"lang" : "en",
"language" : "English",
"parodies" : ["Toaru Kagaku No Railgun"],
"relationships": list,
"tags" : list,
"title" : r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ",
"title_alt" : r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた",
"type" : "Doujinshi",
},
}),
)
def __init__(self, match): def __init__(self, match):
self.slug = match.group(1) self.slug = match.group(1)
@ -76,15 +56,7 @@ class HentaihandTagExtractor(Extractor):
pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com"
r"/\w+/(parody|character|tag|artist|group|language" r"/\w+/(parody|character|tag|artist|group|language"
r"|category|relationship)/([^/?#]+)") r"|category|relationship)/([^/?#]+)")
test = ( example = "https://hentaihand.com/en/tag/TAG"
("https://hentaihand.com/en/artist/takumi-na-muchi", {
"pattern": HentaihandGalleryExtractor.pattern,
"count": ">= 6",
}),
("https://hentaihand.com/en/tag/full-color"),
("https://hentaihand.com/fr/language/japanese"),
("https://hentaihand.com/zh/category/manga"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

48
gallery_dl/extractor/hentaihere.py
View File

@ -23,32 +23,7 @@ class HentaihereChapterExtractor(HentaihereBase, ChapterExtractor):
"""Extractor for a single manga chapter from hentaihere.com""" """Extractor for a single manga chapter from hentaihere.com"""
archive_fmt = "{chapter_id}_{page}" archive_fmt = "{chapter_id}_{page}"
pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com/m/S(\d+)/([^/?#]+)" pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com/m/S(\d+)/([^/?#]+)"
test = ( example = "https://hentaihere.com/m/S12345/1/1/"
("https://hentaihere.com/m/S13812/1/1/", {
"url": "964b942cf492b3a129d2fe2608abfc475bc99e71",
"keyword": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d",
}),
("https://hentaihere.com/m/S23048/1.5/1/", {
"pattern": r"https://hentaicdn\.com/hentai"
r"/23048/1\.5/ccdn00\d+\.jpg",
"count": 32,
"keyword": {
"author": "Shinozuka Yuuji",
"chapter": 1,
"chapter_id": 80186,
"chapter_minor": ".5",
"count": 32,
"lang": "en",
"language": "English",
"manga": "High School Slut's Love Consultation",
"manga_id": 23048,
"page": int,
"title": "High School Slut's Love Consultation + "
"Girlfriend [Full Color]",
"type": "Original",
},
}),
)
def __init__(self, match): def __init__(self, match):
self.manga_id, self.chapter = match.groups() self.manga_id, self.chapter = match.groups()
@ -87,26 +62,7 @@ class HentaihereMangaExtractor(HentaihereBase, MangaExtractor):
"""Extractor for hmanga from hentaihere.com""" """Extractor for hmanga from hentaihere.com"""
chapterclass = HentaihereChapterExtractor chapterclass = HentaihereChapterExtractor
pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com(/m/S\d+)/?$" pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com(/m/S\d+)/?$"
test = ( example = "https://hentaihere.com/m/S12345"
("https://hentaihere.com/m/S13812", {
"url": "d1ba6e28bb2162e844f8559c2b2725ba0a093559",
"keyword": "5c1b712258e78e120907121d3987c71f834d13e1",
}),
("https://hentaihere.com/m/S7608", {
"url": "6c5239758dc93f6b1b4175922836c10391b174f7",
"keyword": {
"chapter": int,
"chapter_id": int,
"chapter_minor": "",
"lang": "en",
"language": "English",
"manga": "Oshikake Riot",
"manga_id": 7608,
"title": r"re:Oshikake Riot( \d+)?",
"type": "Original",
},
}),
)
def chapters(self, page): def chapters(self, page):
results = [] results = []

61
gallery_dl/extractor/hiperdex.py
View File

@ -72,30 +72,7 @@ class HiperdexBase():
class HiperdexChapterExtractor(HiperdexBase, ChapterExtractor): class HiperdexChapterExtractor(HiperdexBase, ChapterExtractor):
"""Extractor for manga chapters from hiperdex.com""" """Extractor for manga chapters from hiperdex.com"""
pattern = BASE_PATTERN + r"(/manga/([^/?#]+)/([^/?#]+))" pattern = BASE_PATTERN + r"(/manga/([^/?#]+)/([^/?#]+))"
test = ( example = "https://hiperdex.com/manga/MANGA/CHAPTER/"
("https://hiperdex.com/manga/domestic-na-kanojo/154-5/", {
"pattern": r"https://(1st)?hiperdex\d?.(com|net|info)"
r"/wp-content/uploads/WP-manga/data"
r"/manga_\w+/[0-9a-f]{32}/\d+\.webp",
"count": 9,
"keyword": {
"artist" : "Sasuga Kei",
"author" : "Sasuga Kei",
"chapter": 154,
"chapter_minor": ".5",
"description": "re:Natsuo Fujii is in love with his teacher, ",
"genre" : list,
"manga" : "Domestic na Kanojo",
"release": 2014,
"score" : float,
"type" : "Manga",
},
}),
("https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/"),
("https://hiperdex2.com/manga/domestic-na-kanojo/154-5/"),
("https://hiperdex.net/manga/domestic-na-kanojo/154-5/"),
("https://hiperdex.info/manga/domestic-na-kanojo/154-5/"),
)
def __init__(self, match): def __init__(self, match):
root, path, self.manga, self.chapter = match.groups() root, path, self.manga, self.chapter = match.groups()
@ -117,30 +94,7 @@ class HiperdexMangaExtractor(HiperdexBase, MangaExtractor):
"""Extractor for manga from hiperdex.com""" """Extractor for manga from hiperdex.com"""
chapterclass = HiperdexChapterExtractor chapterclass = HiperdexChapterExtractor
pattern = BASE_PATTERN + r"(/manga/([^/?#]+))/?$" pattern = BASE_PATTERN + r"(/manga/([^/?#]+))/?$"
test = ( example = "https://hiperdex.com/manga/MANGA/"
("https://hiperdex.com/manga/1603231576-youre-not-that-special/", {
"count": 51,
"pattern": HiperdexChapterExtractor.pattern,
"keyword": {
"artist" : "Bolp",
"author" : "Abyo4",
"chapter": int,
"chapter_minor": "",
"description": "re:I didn’t think much of the creepy girl in ",
"genre" : list,
"manga" : "You’re Not That Special!",
"release": 2019,
"score" : float,
"status" : "Completed",
"type" : "Manhwa",
},
}),
("https://hiperdex.com/manga/youre-not-that-special/"),
("https://1sthiperdex.com/manga/youre-not-that-special/"),
("https://hiperdex2.com/manga/youre-not-that-special/"),
("https://hiperdex.net/manga/youre-not-that-special/"),
("https://hiperdex.info/manga/youre-not-that-special/"),
)
def __init__(self, match): def __init__(self, match):
root, path, self.manga = match.groups() root, path, self.manga = match.groups()
@ -176,16 +130,7 @@ class HiperdexArtistExtractor(HiperdexBase, MangaExtractor):
chapterclass = HiperdexMangaExtractor chapterclass = HiperdexMangaExtractor
reverse = False reverse = False
pattern = BASE_PATTERN + r"(/manga-a(?:rtist|uthor)/(?:[^/?#]+))" pattern = BASE_PATTERN + r"(/manga-a(?:rtist|uthor)/(?:[^/?#]+))"
test = ( example = "https://hiperdex.com/manga-artist/NAME/"
("https://1sthiperdex.com/manga-artist/beck-ho-an/"),
("https://hiperdex.net/manga-artist/beck-ho-an/"),
("https://hiperdex2.com/manga-artist/beck-ho-an/"),
("https://hiperdex.info/manga-artist/beck-ho-an/"),
("https://hiperdex.com/manga-author/viagra/", {
"pattern": HiperdexMangaExtractor.pattern,
"count": ">= 6",
}),
)
def __init__(self, match): def __init__(self, match):
self.root = text.ensure_http_scheme(match.group(1)) self.root = text.ensure_http_scheme(match.group(1))

54
gallery_dl/extractor/hitomi.py
View File

@ -23,47 +23,7 @@ class HitomiGalleryExtractor(GalleryExtractor):
pattern = (r"(?:https?://)?hitomi\.la" pattern = (r"(?:https?://)?hitomi\.la"
r"/(?:manga|doujinshi|cg|gamecg|galleries|reader)" r"/(?:manga|doujinshi|cg|gamecg|galleries|reader)"
r"/(?:[^/?#]+-)?(\d+)") r"/(?:[^/?#]+-)?(\d+)")
test = ( example = "https://hitomi.la/manga/TITLE-867789.html"
("https://hitomi.la/galleries/867789.html", {
"pattern": r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+"
r"/[0-9a-f]{64}\.webp",
"keyword": "86af5371f38117a07407f11af689bdd460b09710",
"count": 16,
}),
# download test
("https://hitomi.la/galleries/1401410.html", {
"range": "1",
"content": "d75d5a3d1302a48469016b20e53c26b714d17745",
}),
# Game CG with scenes (#321)
("https://hitomi.la/galleries/733697.html", {
"count": 210,
}),
# fallback for galleries only available through /reader/ URLs
("https://hitomi.la/galleries/1045954.html", {
"count": 1413,
}),
# gallery with "broken" redirect
("https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", {
"count": 10,
"options": (("format", "original"),),
"pattern": r"https://[a-c]b\.hitomi\.la/images/\d+/\d+"
r"/[0-9a-f]{64}\.jpg",
}),
# no tags
("https://hitomi.la/cg/1615823.html", {
"count": 22,
"options": (("format", "avif"),),
"pattern": r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+"
r"/[0-9a-f]{64}\.avif",
}),
("https://hitomi.la/manga/amazon-no-hiyaku-867789.html"),
("https://hitomi.la/manga/867789.html"),
("https://hitomi.la/doujinshi/867789.html"),
("https://hitomi.la/cg/867789.html"),
("https://hitomi.la/gamecg/867789.html"),
("https://hitomi.la/reader/867789.html"),
)
def __init__(self, match): def __init__(self, match):
self.gid = match.group(1) self.gid = match.group(1)
@ -149,17 +109,7 @@ class HitomiTagExtractor(Extractor):
pattern = (r"(?:https?://)?hitomi\.la/" pattern = (r"(?:https?://)?hitomi\.la/"
r"(tag|artist|group|series|type|character)/" r"(tag|artist|group|series|type|character)/"
r"([^/?#]+)\.html") r"([^/?#]+)\.html")
test = ( example = "https://hitomi.la/tag/TAG-LANG.html"
("https://hitomi.la/tag/screenshots-japanese.html", {
"pattern": HitomiGalleryExtractor.pattern,
"count": ">= 35",
}),
("https://hitomi.la/artist/a1-all-1.html"),
("https://hitomi.la/group/initial%2Dg-all-1.html"),
("https://hitomi.la/series/amnesia-all-1.html"),
("https://hitomi.la/type/doujinshi-all-1.html"),
("https://hitomi.la/character/a2-all-1.html"),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

62
gallery_dl/extractor/hotleak.py
View File

@ -59,30 +59,7 @@ class HotleakPostExtractor(HotleakExtractor):
subcategory = "post" subcategory = "post"
pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))"
r"([^/]+)/(photo|video)/(\d+)") r"([^/]+)/(photo|video)/(\d+)")
test = ( example = "https://hotleak.vip/MODEL/photo/12345"
("https://hotleak.vip/kaiyakawaii/photo/1617145", {
"pattern": r"https://hotleak\.vip/storage/images/3625"
r"/1617145/fefdd5988dfcf6b98cc9e11616018868\.jpg",
"keyword": {
"id": 1617145,
"creator": "kaiyakawaii",
"type": "photo",
"filename": "fefdd5988dfcf6b98cc9e11616018868",
"extension": "jpg",
},
}),
("https://hotleak.vip/lilmochidoll/video/1625538", {
"pattern": r"ytdl:https://cdn8-leak\.camhdxx\.com"
r"/1661/1625538/index\.m3u8",
"keyword": {
"id": 1625538,
"creator": "lilmochidoll",
"type": "video",
"filename": "index",
"extension": "mp4",
},
}),
)
def __init__(self, match): def __init__(self, match):
HotleakExtractor.__init__(self, match) HotleakExtractor.__init__(self, match)
@ -118,18 +95,7 @@ class HotleakCreatorExtractor(HotleakExtractor):
subcategory = "creator" subcategory = "creator"
pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))"
r"([^/?#]+)/?$") r"([^/?#]+)/?$")
test = ( example = "https://hotleak.vip/MODEL"
("https://hotleak.vip/kaiyakawaii", {
"range": "1-200",
"count": 200,
}),
("https://hotleak.vip/stellaviolet", {
"count": "> 600"
}),
("https://hotleak.vip/doesnotexist", {
"exception": exception.NotFoundError,
}),
)
def __init__(self, match): def __init__(self, match):
HotleakExtractor.__init__(self, match) HotleakExtractor.__init__(self, match)
@ -182,20 +148,7 @@ class HotleakCategoryExtractor(HotleakExtractor):
"""Extractor for hotleak categories""" """Extractor for hotleak categories"""
subcategory = "category" subcategory = "category"
pattern = BASE_PATTERN + r"/(hot|creators|videos|photos)(?:/?\?([^#]+))?" pattern = BASE_PATTERN + r"/(hot|creators|videos|photos)(?:/?\?([^#]+))?"
test = ( example = "https://hotleak.vip/photos"
("https://hotleak.vip/photos", {
"pattern": HotleakPostExtractor.pattern,
"range": "1-50",
"count": 50,
}),
("https://hotleak.vip/videos"),
("https://hotleak.vip/creators", {
"pattern": HotleakCreatorExtractor.pattern,
"range": "1-50",
"count": 50,
}),
("https://hotleak.vip/hot"),
)
def __init__(self, match): def __init__(self, match):
HotleakExtractor.__init__(self, match) HotleakExtractor.__init__(self, match)
@ -217,14 +170,7 @@ class HotleakSearchExtractor(HotleakExtractor):
"""Extractor for hotleak search results""" """Extractor for hotleak search results"""
subcategory = "search" subcategory = "search"
pattern = BASE_PATTERN + r"/search(?:/?\?([^#]+))" pattern = BASE_PATTERN + r"/search(?:/?\?([^#]+))"
test = ( example = "https://hotleak.vip/search?search=QUERY"
("https://hotleak.vip/search?search=gallery-dl", {
"count": 0,
}),
("https://hotleak.vip/search?search=hannah", {
"count": "> 30",
}),
)
def __init__(self, match): def __init__(self, match):
HotleakExtractor.__init__(self, match) HotleakExtractor.__init__(self, match)

33
gallery_dl/extractor/idolcomplex.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2018-2021 Mike Fährmann # Copyright 2018-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -133,20 +133,7 @@ class IdolcomplexTagExtractor(IdolcomplexExtractor):
directory_fmt = ("{category}", "{search_tags}") directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}" archive_fmt = "t_{search_tags}_{id}"
pattern = r"(?:https?://)?idol\.sankakucomplex\.com/\?([^#]*)" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/\?([^#]*)"
test = ( example = "https://idol.sankakucomplex.com/?tags=TAGS"
("https://idol.sankakucomplex.com/?tags=lyumos", {
"count": 5,
"range": "18-22",
"pattern": r"https://is\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}"
r"/[^/]{32}\.\w+\?e=\d+&m=[^&#]+",
}),
("https://idol.sankakucomplex.com/?tags=order:favcount", {
"count": 5,
"range": "18-22",
}),
("https://idol.sankakucomplex.com"
"/?tags=lyumos+wreath&page=3&next=694215"),
)
per_page = 20 per_page = 20
def __init__(self, match): def __init__(self, match):
@ -214,9 +201,7 @@ class IdolcomplexPoolExtractor(IdolcomplexExtractor):
directory_fmt = ("{category}", "pool", "{pool}") directory_fmt = ("{category}", "pool", "{pool}")
archive_fmt = "p_{pool}_{id}" archive_fmt = "p_{pool}_{id}"
pattern = r"(?:https?://)?idol\.sankakucomplex\.com/pool/show/(\d+)" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/pool/show/(\d+)"
test = ("https://idol.sankakucomplex.com/pool/show/145", { example = "https://idol.sankakucomplex.com/pool/show/12345"
"count": 3,
})
per_page = 24 per_page = 24
def __init__(self, match): def __init__(self, match):
@ -251,17 +236,7 @@ class IdolcomplexPostExtractor(IdolcomplexExtractor):
subcategory = "post" subcategory = "post"
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = r"(?:https?://)?idol\.sankakucomplex\.com/post/show/(\d+)" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/post/show/(\d+)"
test = ("https://idol.sankakucomplex.com/post/show/694215", { example = "https://idol.sankakucomplex.com/post/show/12345"
"content": "694ec2491240787d75bf5d0c75d0082b53a85afd",
"options": (("tags", True),),
"keyword": {
"tags_character": "shani_(the_witcher)",
"tags_copyright": "the_witcher",
"tags_idol": str,
"tags_medium": str,
"tags_general": str,
},
})
def __init__(self, match): def __init__(self, match):
IdolcomplexExtractor.__init__(self, match) IdolcomplexExtractor.__init__(self, match)

42
gallery_dl/extractor/imagebam.py
View File

@ -9,7 +9,7 @@
"""Extractors for https://www.imagebam.com/""" """Extractors for https://www.imagebam.com/"""
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, exception from .. import text
import re import re
@ -46,26 +46,7 @@ class ImagebamGalleryExtractor(ImagebamExtractor):
archive_fmt = "{gallery_key}_{image_key}" archive_fmt = "{gallery_key}_{image_key}"
pattern = (r"(?:https?://)?(?:www\.)?imagebam\.com" pattern = (r"(?:https?://)?(?:www\.)?imagebam\.com"
r"(/(?:gallery/|view/G)[a-zA-Z0-9]+)") r"(/(?:gallery/|view/G)[a-zA-Z0-9]+)")
test = ( example = "https://www.imagebam.com/view/GID"
("https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", {
"url": "76d976788ae2757ac81694736b07b72356f5c4c8",
"keyword": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c",
"content": "596e6bfa157f2c7169805d50075c2986549973a8",
}),
("http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", {
# more than 100 images; see issue #219
"count": 107,
"url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1",
}),
("http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", {
"exception": exception.HttpError,
}),
# /view/ path (#2378)
("https://www.imagebam.com/view/GA3MT1", {
"url": "35018ce1e00a2d2825a33d3cd37857edaf804919",
"keyword": "3a9f98178f73694c527890c0d7ca9a92b46987ba",
}),
)
def items(self): def items(self):
page = self.request(self.root + self.path).text page = self.request(self.root + self.path).text
@ -110,24 +91,7 @@ class ImagebamImageExtractor(ImagebamExtractor):
archive_fmt = "{image_key}" archive_fmt = "{image_key}"
pattern = (r"(?:https?://)?(?:\w+\.)?imagebam\.com" pattern = (r"(?:https?://)?(?:\w+\.)?imagebam\.com"
r"(/(?:image/|view/M|(?:[0-9a-f]{2}/){3})[a-zA-Z0-9]+)") r"(/(?:image/|view/M|(?:[0-9a-f]{2}/){3})[a-zA-Z0-9]+)")
test = ( example = "https://www.imagebam.com/view/MID"
("https://www.imagebam.com/image/94d56c502511890", {
"url": "5e9ba3b1451f8ded0ae3a1b84402888893915d4a",
"keyword": "2a4380d4b57554ff793898c2d6ec60987c86d1a1",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
}),
("http://images3.imagebam.com/1d/8c/44/94d56c502511890.png"),
# NSFW (#1534)
("https://www.imagebam.com/image/0850951366904951", {
"url": "d37297b17ed1615b4311c8ed511e50ce46e4c748",
}),
# /view/ path (#2378)
("https://www.imagebam.com/view/ME8JOQP", {
"url": "4dca72bbe61a0360185cf4ab2bed8265b49565b8",
"keyword": "15a494c02fd30846b41b42a26117aedde30e4ceb",
"content": "f81008666b17a42d8834c4749b910e1dc10a6e83",
}),
)
def items(self): def items(self):
path = self.path path = self.path

24
gallery_dl/extractor/imagechest.py
View File

@ -18,29 +18,7 @@ class ImagechestGalleryExtractor(GalleryExtractor):
category = "imagechest" category = "imagechest"
root = "https://imgchest.com" root = "https://imgchest.com"
pattern = r"(?:https?://)?(?:www\.)?imgchest\.com/p/([A-Za-z0-9]{11})" pattern = r"(?:https?://)?(?:www\.)?imgchest\.com/p/([A-Za-z0-9]{11})"
test = ( example = "https://imgchest.com/p/abcdefghijk"
("https://imgchest.com/p/3na7kr3by8d", {
"pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)",
"keyword": {
"count": 3,
"gallery_id": "3na7kr3by8d",
"num": int,
"title": "Wizardry - Video Game From The Mid 80's",
},
"url": "7328ca4ec2459378d725e3be19f661d2b045feda",
"content": "076959e65be30249a2c651fbe6090dc30ba85193",
"count": 3
}),
# "Load More Files" button (#4028)
("https://imgchest.com/p/9p4n3q2z7nq", {
"pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)",
"url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7",
"count": 52,
}),
("https://imgchest.com/p/xxxxxxxxxxx", {
"exception": exception.NotFoundError,
}),
)
def __init__(self, match): def __init__(self, match):
self.gallery_id = match.group(1) self.gallery_id = match.group(1)

107
gallery_dl/extractor/imagefap.py
View File

@ -43,50 +43,7 @@ class ImagefapGalleryExtractor(ImagefapExtractor):
"""Extractor for image galleries from imagefap.com""" """Extractor for image galleries from imagefap.com"""
subcategory = "gallery" subcategory = "gallery"
pattern = BASE_PATTERN + r"/(?:gallery\.php\?gid=|gallery/|pictures/)(\d+)" pattern = BASE_PATTERN + r"/(?:gallery\.php\?gid=|gallery/|pictures/)(\d+)"
example = "https://www.imagefap.com/gallery/12345"
test = (
("https://www.imagefap.com/gallery/7102714", {
"pattern": r"https://cdnh?\.imagefap\.com"
r"/images/full/\d+/\d+/\d+\.jpg",
"keyword": "bdcb75b1e4b9dddc718f3d66e1a58afa9d81a38b",
"content": "694a0a57385980a6f90fbc296cadcd6c11ba2dab",
}),
("https://www.imagefap.com/gallery/7876223", {
"pattern": r"https://cdnh?\.imagefap\.com"
r"/images/full/\d+/\d+/\d+\.jpg",
"keyword": {
"categories": ["Asses", "Softcore", "Pornstars"],
"count": 44,
"description": "",
"gallery_id": 7876223,
"image_id": int,
"num": int,
"tags": ["big ass", "panties", "horny",
"pussy", "exposed", "outdoor"],
"title": "Kelsi Monroe in lingerie",
"uploader": "BdRachel",
},
"count": 44,
}),
# description (#3905)
("https://www.imagefap.com/gallery/6180555", {
"range": "1",
"keyword": {
"categories": ["Amateur", "Softcore", "Homemade"],
"count": 36,
"description": "Nude and dressed sluts showing off the goods",
"gallery_id": 6180555,
"image_id": int,
"num": int,
"tags": [] ,
"title": "Dressed or Undressed MG*",
"uploader": "splitopen",
},
}),
("https://www.imagefap.com/pictures/7102714"),
("https://www.imagefap.com/gallery.php?gid=7102714"),
("https://beta.imagefap.com/gallery.php?gid=7102714"),
)
def __init__(self, match): def __init__(self, match):
ImagefapExtractor.__init__(self, match) ImagefapExtractor.__init__(self, match)
@ -157,22 +114,7 @@ class ImagefapImageExtractor(ImagefapExtractor):
"""Extractor for single images from imagefap.com""" """Extractor for single images from imagefap.com"""
subcategory = "image" subcategory = "image"
pattern = BASE_PATTERN + r"/photo/(\d+)" pattern = BASE_PATTERN + r"/photo/(\d+)"
test = ( example = "https://www.imagefap.com/photo/12345"
("https://www.imagefap.com/photo/1962981893", {
"pattern": r"https://cdnh?\.imagefap\.com"
r"/images/full/65/196/1962981893\.jpg",
"keyword": {
"date": "21/08/2014",
"gallery_id": 7876223,
"height": 1600,
"image_id": 1962981893,
"title": "Kelsi Monroe in lingerie",
"uploader": "BdRachel",
"width": 1066,
},
}),
("https://beta.imagefap.com/photo/1962981893"),
)
def __init__(self, match): def __init__(self, match):
ImagefapExtractor.__init__(self, match) ImagefapExtractor.__init__(self, match)
@ -213,35 +155,7 @@ class ImagefapFolderExtractor(ImagefapExtractor):
pattern = (BASE_PATTERN + r"/(?:organizer/|" pattern = (BASE_PATTERN + r"/(?:organizer/|"
r"(?:usergallery\.php\?user(id)?=([^&#]+)&" r"(?:usergallery\.php\?user(id)?=([^&#]+)&"
r"|profile/([^/?#]+)/galleries\?)folderid=)(\d+|-1)") r"|profile/([^/?#]+)/galleries\?)folderid=)(\d+|-1)")
test = ( example = "https://www.imagefap.com/organizer/12345"
("https://www.imagefap.com/organizer/409758", {
"pattern": r"https://www\.imagefap\.com/gallery/7876223",
"url": "37822523e6e4a56feb9dea35653760c86b44ff89",
"count": 1,
}),
(("https://www.imagefap.com/usergallery.php"
"?userid=1981976&folderid=409758"), {
"url": "37822523e6e4a56feb9dea35653760c86b44ff89",
}),
(("https://www.imagefap.com/usergallery.php"
"?user=BdRachel&folderid=409758"), {
"url": "37822523e6e4a56feb9dea35653760c86b44ff89",
}),
("https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", {
"pattern": ImagefapGalleryExtractor.pattern,
"range": "1-40",
}),
(("https://www.imagefap.com/usergallery.php"
"?userid=1981976&folderid=-1"), {
"pattern": ImagefapGalleryExtractor.pattern,
"range": "1-40",
}),
(("https://www.imagefap.com/usergallery.php"
"?user=BdRachel&folderid=-1"), {
"pattern": ImagefapGalleryExtractor.pattern,
"range": "1-40",
}),
)
def __init__(self, match): def __init__(self, match):
ImagefapExtractor.__init__(self, match) ImagefapExtractor.__init__(self, match)
@ -293,20 +207,7 @@ class ImagefapUserExtractor(ImagefapExtractor):
pattern = (BASE_PATTERN + pattern = (BASE_PATTERN +
r"/(?:profile(?:\.php\?user=|/)([^/?#]+)(?:/galleries)?" r"/(?:profile(?:\.php\?user=|/)([^/?#]+)(?:/galleries)?"
r"|usergallery\.php\?userid=(\d+))(?:$|#)") r"|usergallery\.php\?userid=(\d+))(?:$|#)")
test = ( example = "https://www.imagefap.com/profile/USER"
("https://www.imagefap.com/profile/BdRachel", {
"pattern": ImagefapFolderExtractor.pattern,
"count": ">= 18",
}),
("https://www.imagefap.com/usergallery.php?userid=1862791", {
"pattern": r"https://www\.imagefap\.com"
r"/profile/LucyRae/galleries\?folderid=-1",
"count": 1,
}),
("https://www.imagefap.com/profile/BdRachel/galleries"),
("https://www.imagefap.com/profile.php?user=BdRachel"),
("https://beta.imagefap.com/profile.php?user=BdRachel"),
)
def __init__(self, match): def __init__(self, match):
ImagefapExtractor.__init__(self, match) ImagefapExtractor.__init__(self, match)

132
gallery_dl/extractor/imagehosts.py
View File

@ -74,37 +74,7 @@ class ImxtoImageExtractor(ImagehostImageExtractor):
category = "imxto" category = "imxto"
pattern = (r"(?:https?://)?(?:www\.)?((?:imx\.to|img\.yt)" pattern = (r"(?:https?://)?(?:www\.)?((?:imx\.to|img\.yt)"
r"/(?:i/|img-)(\w+)(\.html)?)") r"/(?:i/|img-)(\w+)(\.html)?)")
test = ( example = "https://imx.to/i/ID"
# new-style URL
("https://imx.to/i/1qdeva", {
"url": "ab2173088a6cdef631d7a47dec4a5da1c6a00130",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
"keyword": {
"size" : 18,
"width" : 64,
"height": 32,
"hash" : "94d56c599223c59f3feb71ea603484d1",
},
}),
# old-style URL
("https://imx.to/img-57a2050547b97.html", {
"url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204",
"content": "54592f2635674c25677c6872db3709d343cdf92f",
"keyword": {
"size" : 5284,
"width" : 320,
"height": 160,
"hash" : "40da6aaa7b8c42b18ef74309bbc713fc",
},
}),
# img.yt domain
("https://img.yt/img-57a2050547b97.html", {
"url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204",
}),
("https://imx.to/img-57a2050547b98.html", {
"exception": exception.NotFoundError,
}),
)
_params = "simple" _params = "simple"
_encoding = "utf-8" _encoding = "utf-8"
@ -143,11 +113,7 @@ class ImxtoGalleryExtractor(ImagehostImageExtractor):
category = "imxto" category = "imxto"
subcategory = "gallery" subcategory = "gallery"
pattern = r"(?:https?://)?(?:www\.)?(imx\.to/g/([^/?#]+))" pattern = r"(?:https?://)?(?:www\.)?(imx\.to/g/([^/?#]+))"
test = ("https://imx.to/g/ozdy", { example = "https://imx.to/g/ID"
"pattern": ImxtoImageExtractor.pattern,
"keyword": {"title": "untitled gallery"},
"count": 40,
})
def items(self): def items(self):
page = self.request(self.page_url).text page = self.request(self.page_url).text
@ -165,11 +131,7 @@ class AcidimgImageExtractor(ImagehostImageExtractor):
"""Extractor for single images from acidimg.cc""" """Extractor for single images from acidimg.cc"""
category = "acidimg" category = "acidimg"
pattern = r"(?:https?://)?((?:www\.)?acidimg\.cc/img-([a-z0-9]+)\.html)" pattern = r"(?:https?://)?((?:www\.)?acidimg\.cc/img-([a-z0-9]+)\.html)"
test = ("https://acidimg.cc/img-5acb6b9de4640.html", { example = "https://acidimg.cc/img-abc123.html"
"url": "f132a630006e8d84f52d59555191ed82b3b64c04",
"keyword": "135347ab4345002fc013863c0d9419ba32d98f78",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
})
_params = "simple" _params = "simple"
_encoding = "utf-8" _encoding = "utf-8"
@ -192,22 +154,7 @@ class ImagevenueImageExtractor(ImagehostImageExtractor):
category = "imagevenue" category = "imagevenue"
pattern = (r"(?:https?://)?((?:www|img\d+)\.imagevenue\.com" pattern = (r"(?:https?://)?((?:www|img\d+)\.imagevenue\.com"
r"/([A-Z0-9]{8,10}|view/.*|img\.php\?.*))") r"/([A-Z0-9]{8,10}|view/.*|img\.php\?.*))")
test = ( example = "https://www.imagevenue.com/ME123456789"
("https://www.imagevenue.com/ME13LS07", {
"pattern": r"https://cdn-images\.imagevenue\.com"
r"/10/ac/05/ME13LS07_o\.png",
"keyword": "ae15d6e3b2095f019eee84cd896700cd34b09c36",
"content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee",
}),
(("https://www.imagevenue.com/view/o?i=92518_13732377"
"annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486"), {
"url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980",
}),
(("http://img28116.imagevenue.com/img.php"
"?image=th_52709_test_122_64lo.jpg"), {
"url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331",
}),
)
def get_info(self, page): def get_info(self, page):
pos = page.index('class="card-body') pos = page.index('class="card-body')
@ -223,17 +170,7 @@ class ImagetwistImageExtractor(ImagehostImageExtractor):
category = "imagetwist" category = "imagetwist"
pattern = (r"(?:https?://)?((?:www\.|phun\.)?" pattern = (r"(?:https?://)?((?:www\.|phun\.)?"
r"image(?:twist|haha)\.com/([a-z0-9]{12}))") r"image(?:twist|haha)\.com/([a-z0-9]{12}))")
test = ( example = "https://imagetwist.com/123456abcdef/NAME.EXT"
("https://imagetwist.com/f1i2s4vhvbrq/test.png", {
"url": "8d5e168c0bee30211f821c6f3b2116e419d42671",
"keyword": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
}),
("https://www.imagetwist.com/f1i2s4vhvbrq/test.png"),
("https://phun.imagetwist.com/f1i2s4vhvbrq/test.png"),
("https://imagehaha.com/f1i2s4vhvbrq/test.png"),
("https://www.imagehaha.com/f1i2s4vhvbrq/test.png"),
)
@property @property
@memcache(maxage=3*3600) @memcache(maxage=3*3600)
@ -250,11 +187,7 @@ class ImgspiceImageExtractor(ImagehostImageExtractor):
"""Extractor for single images from imgspice.com""" """Extractor for single images from imgspice.com"""
category = "imgspice" category = "imgspice"
pattern = r"(?:https?://)?((?:www\.)?imgspice\.com/([^/?#]+))" pattern = r"(?:https?://)?((?:www\.)?imgspice\.com/([^/?#]+))"
test = ("https://imgspice.com/nwfwtpyog50y/test.png.html", { example = "https://imgspice.com/ID/NAME.EXT.html"
"url": "b8c30a8f51ee1012959a4cfd46197fabf14de984",
"keyword": "100e310a19a2fa22d87e1bbc427ecb9f6501e0c0",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
})
def get_info(self, page): def get_info(self, page):
pos = page.find('id="imgpreview"') pos = page.find('id="imgpreview"')
@ -270,11 +203,7 @@ class PixhostImageExtractor(ImagehostImageExtractor):
category = "pixhost" category = "pixhost"
pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)" pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)"
r"/show/\d+/(\d+)_[^/?#]+)") r"/show/\d+/(\d+)_[^/?#]+)")
test = ("https://pixhost.to/show/190/130327671_test-.png", { example = "https://pixhost.to/show/123/12345_NAME.EXT"
"url": "4e5470dcf6513944773044d40d883221bbc46cff",
"keyword": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
})
_cookies = {"pixhostads": "1", "pixhosttest": "1"} _cookies = {"pixhostads": "1", "pixhosttest": "1"}
def get_info(self, page): def get_info(self, page):
@ -289,10 +218,7 @@ class PixhostGalleryExtractor(ImagehostImageExtractor):
subcategory = "gallery" subcategory = "gallery"
pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)" pattern = (r"(?:https?://)?((?:www\.)?pixhost\.(?:to|org)"
r"/gallery/([^/?#]+))") r"/gallery/([^/?#]+))")
test = ("https://pixhost.to/gallery/jSMFq", { example = "https://pixhost.to/gallery/ID"
"pattern": PixhostImageExtractor.pattern,
"count": 3,
})
def items(self): def items(self):
page = text.extr(self.request( page = text.extr(self.request(
@ -307,16 +233,7 @@ class PostimgImageExtractor(ImagehostImageExtractor):
category = "postimg" category = "postimg"
pattern = (r"(?:https?://)?((?:www\.)?(?:postim(?:ages|g)|pixxxels)" pattern = (r"(?:https?://)?((?:www\.)?(?:postim(?:ages|g)|pixxxels)"
r"\.(?:cc|org)/(?!gallery/)(?:image/)?([^/?#]+)/?)") r"\.(?:cc|org)/(?!gallery/)(?:image/)?([^/?#]+)/?)")
test = ( example = "https://postimages.org/ID"
("https://postimages.org/Wtn2b3hC"),
("https://www.postimages.org/Wtn2b3hC"),
("https://pixxxels.cc/Wtn2b3hC"),
("https://postimg.cc/Wtn2b3hC", {
"url": "72f3c8b1d6c6601a20ad58f35635494b4891a99e",
"keyword": "2d05808d04e4e83e33200db83521af06e3147a84",
"content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee",
}),
)
def get_info(self, page): def get_info(self, page):
pos = page.index(' id="download"') pos = page.index(' id="download"')
@ -331,10 +248,7 @@ class PostimgGalleryExtractor(ImagehostImageExtractor):
subcategory = "gallery" subcategory = "gallery"
pattern = (r"(?:https?://)?((?:www\.)?(?:postimg|pixxxels)\.(?:cc|org)" pattern = (r"(?:https?://)?((?:www\.)?(?:postimg|pixxxels)\.(?:cc|org)"
r"/(?:gallery/)([^/?#]+)/?)") r"/(?:gallery/)([^/?#]+)/?)")
test = ("https://postimg.cc/gallery/wxpDLgX", { example = "https://postimages.org/gallery/ID"
"pattern": PostimgImageExtractor.pattern,
"count": 22,
})
def items(self): def items(self):
page = self.request(self.page_url).text page = self.request(self.page_url).text
@ -348,11 +262,7 @@ class TurboimagehostImageExtractor(ImagehostImageExtractor):
category = "turboimagehost" category = "turboimagehost"
pattern = (r"(?:https?://)?((?:www\.)?turboimagehost\.com" pattern = (r"(?:https?://)?((?:www\.)?turboimagehost\.com"
r"/p/(\d+)/[^/?#]+\.html)") r"/p/(\d+)/[^/?#]+\.html)")
test = ("https://www.turboimagehost.com/p/39078423/test--.png.html", { example = "https://www.turboimagehost.com/p/12345/NAME.EXT.html"
"url": "b94de43612318771ced924cb5085976f13b3b90e",
"keyword": "704757ca8825f51cec516ec44c1e627c1f2058ca",
"content": "f38b54b17cd7462e687b58d83f00fca88b1b105a",
})
def get_info(self, page): def get_info(self, page):
url = text.extract(page, 'src="', '"', page.index("<img "))[0] url = text.extract(page, 'src="', '"', page.index("<img "))[0]
@ -363,10 +273,7 @@ class ViprImageExtractor(ImagehostImageExtractor):
"""Extractor for single images from vipr.im""" """Extractor for single images from vipr.im"""
category = "vipr" category = "vipr"
pattern = r"(?:https?://)?(vipr\.im/(\w+))" pattern = r"(?:https?://)?(vipr\.im/(\w+))"
test = ("https://vipr.im/kcd5jcuhgs3v.html", { example = "https://vipr.im/abc123.html"
"url": "88f6a3ecbf3356a11ae0868b518c60800e070202",
"keyword": "c432e8a1836b0d97045195b745731c2b1bb0e771",
})
def get_info(self, page): def get_info(self, page):
url = text.extr(page, '<img src="', '"') url = text.extr(page, '<img src="', '"')
@ -377,11 +284,7 @@ class ImgclickImageExtractor(ImagehostImageExtractor):
"""Extractor for single images from imgclick.net""" """Extractor for single images from imgclick.net"""
category = "imgclick" category = "imgclick"
pattern = r"(?:https?://)?((?:www\.)?imgclick\.net/([^/?#]+))" pattern = r"(?:https?://)?((?:www\.)?imgclick\.net/([^/?#]+))"
test = ("http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", { example = "http://imgclick.net/abc123/NAME.EXT.html"
"url": "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab",
"keyword": "6895256143eab955622fc149aa367777a8815ba3",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
})
_https = False _https = False
_params = "complex" _params = "complex"
@ -395,14 +298,7 @@ class FappicImageExtractor(ImagehostImageExtractor):
"""Extractor for single images from fappic.com""" """Extractor for single images from fappic.com"""
category = "fappic" category = "fappic"
pattern = r"(?:https?://)?((?:www\.)?fappic\.com/(\w+)/[^/?#]+)" pattern = r"(?:https?://)?((?:www\.)?fappic\.com/(\w+)/[^/?#]+)"
test = ( example = "https://fappic.com/abc123/NAME.EXT"
("https://fappic.com/98wxqcklyh8k/test.png"),
("https://www.fappic.com/98wxqcklyh8k/test.png", {
"pattern": r"https://img\d+\.fappic\.com/img/\w+/test\.png",
"keyword": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
}),
)
def get_info(self, page): def get_info(self, page):
url , pos = text.extract(page, '<a href="#"><img src="', '"') url , pos = text.extract(page, '<a href="#"><img src="', '"')

43
gallery_dl/extractor/imgbb.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2019 Mike Fährmann # Copyright 2019-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -114,27 +114,7 @@ class ImgbbAlbumExtractor(ImgbbExtractor):
subcategory = "album" subcategory = "album"
directory_fmt = ("{category}", "{user}", "{album_name} {album_id}") directory_fmt = ("{category}", "{user}", "{album_name} {album_id}")
pattern = r"(?:https?://)?ibb\.co/album/([^/?#]+)/?(?:\?([^#]+))?" pattern = r"(?:https?://)?ibb\.co/album/([^/?#]+)/?(?:\?([^#]+))?"
test = ( example = "https://ibb.co/album/ID"
("https://ibb.co/album/i5PggF", {
"range": "1-80",
"url": "70afec9fcc3a6de62a6b644b487d892d8d47cf1a",
"keyword": "569e1d88ebdd27655387559cdf1cd526a3e1ab69",
}),
("https://ibb.co/album/i5PggF?sort=title_asc", {
"range": "1-80",
"url": "afdf5fc95d8e09d77e8f44312f3e9b843987bb5a",
"keyword": "f090e14d0e5f7868595082b2c95da1309c84872d",
}),
# no user data (#471)
("https://ibb.co/album/kYKpwF", {
"url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7",
"keyword": {"user": ""},
}),
# private
("https://ibb.co/album/hqgWrF", {
"exception": exception.HttpError,
}),
)
def __init__(self, match): def __init__(self, match):
ImgbbExtractor.__init__(self, match) ImgbbExtractor.__init__(self, match)
@ -169,10 +149,7 @@ class ImgbbUserExtractor(ImgbbExtractor):
"""Extractor for user profiles in imgbb.com""" """Extractor for user profiles in imgbb.com"""
subcategory = "user" subcategory = "user"
pattern = r"(?:https?://)?([\w-]+)\.imgbb\.com/?(?:\?([^#]+))?$" pattern = r"(?:https?://)?([\w-]+)\.imgbb\.com/?(?:\?([^#]+))?$"
test = ("https://folkie.imgbb.com", { example = "https://USER.imgbb.com"
"range": "1-80",
"pattern": r"https?://i\.ibb\.co/\w+/[^/?#]+",
})
def __init__(self, match): def __init__(self, match):
ImgbbExtractor.__init__(self, match) ImgbbExtractor.__init__(self, match)
@ -196,19 +173,7 @@ class ImgbbUserExtractor(ImgbbExtractor):
class ImgbbImageExtractor(ImgbbExtractor): class ImgbbImageExtractor(ImgbbExtractor):
subcategory = "image" subcategory = "image"
pattern = r"(?:https?://)?ibb\.co/(?!album/)([^/?#]+)" pattern = r"(?:https?://)?ibb\.co/(?!album/)([^/?#]+)"
test = ("https://ibb.co/fUqh5b", { example = "https://ibb.co/ID"
"pattern": r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg",
"content": "c5a0965178a8b357acd8aa39660092918c63795e",
"keyword": {
"id" : "fUqh5b",
"title" : "Arundel Ireeman 5",
"url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg",
"width" : 960,
"height": 719,
"user" : "folkie",
"extension": "jpg",
},
})
def __init__(self, match): def __init__(self, match):
ImgbbExtractor.__init__(self, match) ImgbbExtractor.__init__(self, match)

30
gallery_dl/extractor/imgbox.py
View File

@ -1,12 +1,12 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2014-2019 Mike Fährmann # Copyright 2014-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation. # published by the Free Software Foundation.
"""Extract images from galleries at https://imgbox.com/""" """Extractors for https://imgbox.com/"""
from .common import Extractor, Message, AsynchronousMixin from .common import Extractor, Message, AsynchronousMixin
from .. import text, exception from .. import text, exception
@ -63,20 +63,7 @@ class ImgboxGalleryExtractor(AsynchronousMixin, ImgboxExtractor):
filename_fmt = "{num:>03}-{filename}.{extension}" filename_fmt = "{num:>03}-{filename}.{extension}"
archive_fmt = "{gallery_key}_{image_key}" archive_fmt = "{gallery_key}_{image_key}"
pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/g/([A-Za-z0-9]{10})" pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/g/([A-Za-z0-9]{10})"
test = ( example = "https://imgbox.com/g/12345abcde"
("https://imgbox.com/g/JaX5V5HX7g", {
"url": "da4f15b161461119ee78841d4b8e8d054d95f906",
"keyword": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c",
"content": "d20307dc8511ac24d688859c55abf2e2cc2dd3cc",
}),
("https://imgbox.com/g/cUGEkRbdZZ", {
"url": "76506a3aab175c456910851f66227e90484ca9f7",
"keyword": "fb0427b87983197849fb2887905e758f3e50cb6e",
}),
("https://imgbox.com/g/JaX5V5HX7h", {
"exception": exception.NotFoundError,
}),
)
def __init__(self, match): def __init__(self, match):
ImgboxExtractor.__init__(self, match) ImgboxExtractor.__init__(self, match)
@ -106,16 +93,7 @@ class ImgboxImageExtractor(ImgboxExtractor):
subcategory = "image" subcategory = "image"
archive_fmt = "{image_key}" archive_fmt = "{image_key}"
pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/([A-Za-z0-9]{8})" pattern = r"(?:https?://)?(?:www\.)?imgbox\.com/([A-Za-z0-9]{8})"
test = ( example = "https://imgbox.com/1234abcd"
("https://imgbox.com/qHhw7lpG", {
"url": "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c",
"keyword": "dfc72310026b45f3feb4f9cada20c79b2575e1af",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
}),
("https://imgbox.com/qHhw7lpH", {
"exception": exception.NotFoundError,
}),
)
def __init__(self, match): def __init__(self, match):
ImgboxExtractor.__init__(self, match) ImgboxExtractor.__init__(self, match)

21
gallery_dl/extractor/imgth.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2015-2022 Mike Fährmann # Copyright 2015-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -17,24 +17,7 @@ class ImgthGalleryExtractor(GalleryExtractor):
category = "imgth" category = "imgth"
root = "https://imgth.com" root = "https://imgth.com"
pattern = r"(?:https?://)?(?:www\.)?imgth\.com/gallery/(\d+)" pattern = r"(?:https?://)?(?:www\.)?imgth\.com/gallery/(\d+)"
test = ( example = "https://imgth.com/gallery/123/TITLE"
("https://imgth.com/gallery/37/wallpaper-anime", {
"url": "4ae1d281ca2b48952cf5cca57e9914402ad72748",
"pattern": r"https://imgth\.com/images/2009/11/25"
r"/wallpaper-anime_\w+\.jpg",
"keyword": {
"count": 12,
"date": "dt:2009-11-25 18:21:00",
"extension": "jpg",
"filename": r"re:wallpaper-anime_\w+",
"gallery_id": 37,
"num": int,
"title": "Wallpaper anime",
"user": "celebrities",
},
}),
("https://www.imgth.com/gallery/37/wallpaper-anime"),
)
def __init__(self, match): def __init__(self, match):
self.gallery_id = gid = match.group(1) self.gallery_id = gid = match.group(1)

209
gallery_dl/extractor/imgur.py
View File

@ -11,7 +11,6 @@
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, exception from .. import text, exception
BASE_PATTERN = r"(?:https?://)?(?:www\.|[im]\.)?imgur\.(?:com|io)" BASE_PATTERN = r"(?:https?://)?(?:www\.|[im]\.)?imgur\.(?:com|io)"
@ -65,82 +64,7 @@ class ImgurImageExtractor(ImgurExtractor):
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = (BASE_PATTERN + r"/(?!gallery|search)" pattern = (BASE_PATTERN + r"/(?!gallery|search)"
r"(?:r/\w+/)?(\w{7}|\w{5})[sbtmlh]?") r"(?:r/\w+/)?(\w{7}|\w{5})[sbtmlh]?")
test = ( example = "https://imgur.com/abcdefg"
("https://imgur.com/21yMxCS", {
"url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
"keyword": {
"account_id" : 0,
"comment_count" : int,
"cover_id" : "21yMxCS",
"date" : "dt:2016-11-10 14:24:35",
"description" : "",
"downvote_count": int,
"duration" : 0,
"ext" : "png",
"favorite" : False,
"favorite_count": 0,
"has_sound" : False,
"height" : 32,
"id" : "21yMxCS",
"image_count" : 1,
"in_most_viral" : False,
"is_ad" : False,
"is_album" : False,
"is_animated" : False,
"is_looping" : False,
"is_mature" : False,
"is_pending" : False,
"mime_type" : "image/png",
"name" : "test-テスト",
"point_count" : int,
"privacy" : "",
"score" : int,
"size" : 182,
"title" : "Test",
"upvote_count" : int,
"url" : "https://i.imgur.com/21yMxCS.png",
"view_count" : int,
"width" : 64,
},
}),
# gifv/mp4 video
("http://imgur.com/0gybAXR", {
"url": "a2220eb265a55b0c95e0d3d721ec7665460e3fd7",
"content": "a3c080e43f58f55243ab830569ba02309d59abfc",
}),
# missing title in API response (#467)
("https://imgur.com/XFfsmuC", {
"keyword": {"title": "Tears are a natural response to irritants"},
}),
# animated png
("https://imgur.com/1Nily2P", {
"pattern": "https://i.imgur.com/1Nily2P.png",
}),
# not found
("https://imgur.com/zzzzzzz", {
"exception": exception.HttpError,
}),
("https://m.imgur.com/r/Celebs/iHJ7tsM"),
# www
("https://www.imgur.com/21yMxCS"),
# mobile
("https://m.imgur.com/21yMxCS"),
# 5 character key
("https://imgur.com/zxaY6"),
# .io
("https://imgur.io/zxaY6"),
# direct link
("https://i.imgur.com/21yMxCS.png"),
# direct link .io
("https://i.imgur.io/21yMxCS.png"),
# direct link thumbnail
("https://i.imgur.com/21yMxCSh.png"),
# direct link (short)
("https://i.imgur.com/zxaY6.gif"),
# direct link (short; thumb)
("https://i.imgur.com/zxaY6s.gif"),
)
def items(self): def items(self):
image = self.api.image(self.key) image = self.api.image(self.key)
@ -165,77 +89,7 @@ class ImgurAlbumExtractor(ImgurExtractor):
filename_fmt = "{category}_{album[id]}_{num:>03}_{id}.{extension}" filename_fmt = "{category}_{album[id]}_{num:>03}_{id}.{extension}"
archive_fmt = "{album[id]}_{id}" archive_fmt = "{album[id]}_{id}"
pattern = BASE_PATTERN + r"/a/(\w{7}|\w{5})" pattern = BASE_PATTERN + r"/a/(\w{7}|\w{5})"
test = ( example = "https://imgur.com/a/abcde"
("https://imgur.com/a/TcBmP", {
"url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563",
"keyword": {
"album": {
"account_id" : 0,
"comment_count" : int,
"cover_id" : "693j2Kr",
"date" : "dt:2015-10-09 10:37:50",
"description" : "",
"downvote_count": 0,
"favorite" : False,
"favorite_count": 0,
"id" : "TcBmP",
"image_count" : 19,
"in_most_viral" : False,
"is_ad" : False,
"is_album" : True,
"is_mature" : False,
"is_pending" : False,
"privacy" : "private",
"score" : int,
"title" : "138",
"upvote_count" : int,
"url" : "https://imgur.com/a/TcBmP",
"view_count" : int,
"virality" : int,
},
"account_id" : 0,
"count" : 19,
"date" : "type:datetime",
"description": "",
"ext" : "jpg",
"has_sound" : False,
"height" : int,
"id" : str,
"is_animated": False,
"is_looping" : False,
"mime_type" : "image/jpeg",
"name" : str,
"num" : int,
"size" : int,
"title" : str,
"type" : "image",
"updated_at" : None,
"url" : str,
"width" : int,
},
}),
# large album
("https://imgur.com/a/eD9CT", {
"url": "de748c181a04d18bef1de9d4f4866ef0a06d632b",
}),
# 7 character album hash
("https://imgur.com/a/RhJXhVT/all", {
"url": "695ef0c950023362a0163ee5041796300db76674",
}),
("https://imgur.com/a/TcBmQ", {
"exception": exception.HttpError,
}),
# empty, no 'media' (#2557)
("https://imgur.com/a/pjOnJA0", {
"count": 0,
}),
# www
("https://www.imgur.com/a/TcBmP"),
# .io
("https://imgur.io/a/TcBmP"),
# mobile
("https://m.imgur.com/a/TcBmP"),
)
def items(self): def items(self):
album = self.api.album(self.key) album = self.api.album(self.key)
@ -268,19 +122,7 @@ class ImgurGalleryExtractor(ImgurExtractor):
"""Extractor for imgur galleries""" """Extractor for imgur galleries"""
subcategory = "gallery" subcategory = "gallery"
pattern = BASE_PATTERN + r"/(?:gallery|t/\w+)/(\w{7}|\w{5})" pattern = BASE_PATTERN + r"/(?:gallery|t/\w+)/(\w{7}|\w{5})"
test = ( example = "https://imgur.com/gallery/abcde"
# non-album gallery (#380)
("https://imgur.com/gallery/zf2fIms", {
"pattern": "https://imgur.com/zf2fIms",
}),
("https://imgur.com/gallery/eD9CT", {
"pattern": "https://imgur.com/a/eD9CT",
}),
("https://imgur.com/t/unmuted/26sEhNr"),
("https://imgur.com/t/cat/qSB8NbN"),
# .io
("https://imgur.io/t/cat/qSB8NbN"),
)
def items(self): def items(self):
if self.api.gallery(self.key)["is_album"]: if self.api.gallery(self.key)["is_album"]:
@ -296,15 +138,7 @@ class ImgurUserExtractor(ImgurExtractor):
"""Extractor for all images posted by a user""" """Extractor for all images posted by a user"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/user/([^/?#]+)(?:/posts|/submitted)?/?$" pattern = BASE_PATTERN + r"/user/([^/?#]+)(?:/posts|/submitted)?/?$"
test = ( example = "https://imgur.com/user/USER"
("https://imgur.com/user/Miguenzo", {
"range": "1-100",
"count": 100,
"pattern": r"https://imgur\.com(/a)?/\w+$",
}),
("https://imgur.com/user/Miguenzo/posts"),
("https://imgur.com/user/Miguenzo/submitted"),
)
def items(self): def items(self):
return self._items_queue(self.api.account_submissions(self.key)) return self._items_queue(self.api.account_submissions(self.key))
@ -314,11 +148,7 @@ class ImgurFavoriteExtractor(ImgurExtractor):
"""Extractor for a user's favorites""" """Extractor for a user's favorites"""
subcategory = "favorite" subcategory = "favorite"
pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/?$" pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/?$"
test = ("https://imgur.com/user/Miguenzo/favorites", { example = "https://imgur.com/user/USER/favorites"
"range": "1-100",
"count": 100,
"pattern": r"https://imgur\.com(/a)?/\w+$",
})
def items(self): def items(self):
return self._items_queue(self.api.account_favorites(self.key)) return self._items_queue(self.api.account_favorites(self.key))
@ -328,16 +158,7 @@ class ImgurFavoriteFolderExtractor(ImgurExtractor):
"""Extractor for a user's favorites folder""" """Extractor for a user's favorites folder"""
subcategory = "favorite-folder" subcategory = "favorite-folder"
pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/folder/(\d+)" pattern = BASE_PATTERN + r"/user/([^/?#]+)/favorites/folder/(\d+)"
test = ( example = "https://imgur.com/user/USER/favorites/folder/12345/TITLE"
("https://imgur.com/user/mikf1/favorites/folder/11896757/public", {
"pattern": r"https://imgur\.com(/a)?/\w+$",
"count": 3,
}),
("https://imgur.com/user/mikf1/favorites/folder/11896741/private", {
"pattern": r"https://imgur\.com(/a)?/\w+$",
"count": 5,
}),
)
def __init__(self, match): def __init__(self, match):
ImgurExtractor.__init__(self, match) ImgurExtractor.__init__(self, match)
@ -352,11 +173,7 @@ class ImgurSubredditExtractor(ImgurExtractor):
"""Extractor for a subreddits's imgur links""" """Extractor for a subreddits's imgur links"""
subcategory = "subreddit" subcategory = "subreddit"
pattern = BASE_PATTERN + r"/r/([^/?#]+)/?$" pattern = BASE_PATTERN + r"/r/([^/?#]+)/?$"
test = ("https://imgur.com/r/pics", { example = "https://imgur.com/r/SUBREDDIT"
"range": "1-100",
"count": 100,
"pattern": r"https://imgur\.com(/a)?/\w+$",
})
def items(self): def items(self):
return self._items_queue(self.api.gallery_subreddit(self.key)) return self._items_queue(self.api.gallery_subreddit(self.key))
@ -366,11 +183,7 @@ class ImgurTagExtractor(ImgurExtractor):
"""Extractor for imgur tag searches""" """Extractor for imgur tag searches"""
subcategory = "tag" subcategory = "tag"
pattern = BASE_PATTERN + r"/t/([^/?#]+)$" pattern = BASE_PATTERN + r"/t/([^/?#]+)$"
test = ("https://imgur.com/t/animals", { example = "https://imgur.com/t/TAG"
"range": "1-100",
"count": 100,
"pattern": r"https://imgur\.com(/a)?/\w+$",
})
def items(self): def items(self):
return self._items_queue(self.api.gallery_tag(self.key)) return self._items_queue(self.api.gallery_tag(self.key))
@ -380,11 +193,7 @@ class ImgurSearchExtractor(ImgurExtractor):
"""Extractor for imgur search results""" """Extractor for imgur search results"""
subcategory = "search" subcategory = "search"
pattern = BASE_PATTERN + r"/search(?:/[^?#]+)?/?\?q=([^&#]+)" pattern = BASE_PATTERN + r"/search(?:/[^?#]+)?/?\?q=([^&#]+)"
test = ("https://imgur.com/search?q=cute+cat", { example = "https://imgur.com/search?q=UERY"
"range": "1-100",
"count": 100,
"pattern": r"https://imgur\.com(/a)?/\w+$",
})
def items(self): def items(self):
key = text.unquote(self.key.replace("+", " ")) key = text.unquote(self.key.replace("+", " "))

113
gallery_dl/extractor/inkbunny.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2020-2022 Mike Fährmann # Copyright 2020-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -72,51 +72,7 @@ class InkbunnyUserExtractor(InkbunnyExtractor):
"""Extractor for inkbunny user profiles""" """Extractor for inkbunny user profiles"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/(?!s/)(gallery/|scraps/)?(\w+)(?:$|[/?#])" pattern = BASE_PATTERN + r"/(?!s/)(gallery/|scraps/)?(\w+)(?:$|[/?#])"
test = ( example = "https://inkbunny.net/USER"
("https://inkbunny.net/soina", {
"pattern": r"https://[\w.]+\.metapix\.net/files/full"
r"/\d+/\d+_soina_.+",
"range": "20-50",
"keyword": {
"date" : "type:datetime",
"deleted" : bool,
"file_id" : "re:[0-9]+",
"filename" : r"re:[0-9]+_soina_\w+",
"full_file_md5": "re:[0-9a-f]{32}",
"mimetype" : str,
"submission_id": "re:[0-9]+",
"user_id" : "20969",
"comments_count" : "re:[0-9]+",
"deleted" : bool,
"favorite" : bool,
"favorites_count": "re:[0-9]+",
"friends_only" : bool,
"guest_block" : bool,
"hidden" : bool,
"pagecount" : "re:[0-9]+",
"pools" : list,
"pools_count" : int,
"public" : bool,
"rating_id" : "re:[0-9]+",
"rating_name" : str,
"ratings" : list,
"scraps" : bool,
"tags" : list,
"title" : str,
"type_name" : str,
"username" : "soina",
"views" : str,
},
}),
("https://inkbunny.net/gallery/soina", {
"range": "1-25",
"keyword": {"scraps": False},
}),
("https://inkbunny.net/scraps/soina", {
"range": "1-25",
"keyword": {"scraps": True},
}),
)
def __init__(self, match): def __init__(self, match):
kind, self.user = match.groups() kind, self.user = match.groups()
@ -148,14 +104,7 @@ class InkbunnyPoolExtractor(InkbunnyExtractor):
pattern = (BASE_PATTERN + r"/(?:" pattern = (BASE_PATTERN + r"/(?:"
r"poolview_process\.php\?pool_id=(\d+)|" r"poolview_process\.php\?pool_id=(\d+)|"
r"submissionsviewall\.php\?([^#]+&mode=pool&[^#]+))") r"submissionsviewall\.php\?([^#]+&mode=pool&[^#]+))")
test = ( example = "https://inkbunny.net/poolview_process.php?pool_id=12345"
("https://inkbunny.net/poolview_process.php?pool_id=28985", {
"count": 9,
"keyword": {"pool_id": "28985"},
}),
("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff"
"&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no"),
)
def __init__(self, match): def __init__(self, match):
InkbunnyExtractor.__init__(self, match) InkbunnyExtractor.__init__(self, match)
@ -185,16 +134,8 @@ class InkbunnyFavoriteExtractor(InkbunnyExtractor):
pattern = (BASE_PATTERN + r"/(?:" pattern = (BASE_PATTERN + r"/(?:"
r"userfavorites_process\.php\?favs_user_id=(\d+)|" r"userfavorites_process\.php\?favs_user_id=(\d+)|"
r"submissionsviewall\.php\?([^#]+&mode=userfavs&[^#]+))") r"submissionsviewall\.php\?([^#]+&mode=userfavs&[^#]+))")
test = ( example = ("https://inkbunny.net/userfavorites_process.php"
("https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", { "?mode=userfavs&favs_user_id=12345")
"pattern": r"https://[\w.]+\.metapix\.net/files/full"
r"/\d+/\d+_\w+_.+",
"range": "20-50",
"keyword": {"favs_user_id": "20969"},
}),
("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff"
"&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969"),
)
def __init__(self, match): def __init__(self, match):
InkbunnyExtractor.__init__(self, match) InkbunnyExtractor.__init__(self, match)
@ -225,26 +166,8 @@ class InkbunnySearchExtractor(InkbunnyExtractor):
subcategory = "search" subcategory = "search"
pattern = (BASE_PATTERN + pattern = (BASE_PATTERN +
r"/submissionsviewall\.php\?([^#]+&mode=search&[^#]+)") r"/submissionsviewall\.php\?([^#]+&mode=search&[^#]+)")
test = (("https://inkbunny.net/submissionsviewall.php?rid=ffffffffff" example = ("https://inkbunny.net/submissionsviewall.php"
"&mode=search&page=1&orderby=create_datetime&text=cute" "?mode=search&text=TAG")
"&stringtype=and&keywords=yes&title=yes&description=no&artist="
"&favsby=&type=&days=&keyword_id=&user_id=&random=&md5="), {
"range": "1-10",
"count": 10,
"keyword": {
"search": {
"rid": "ffffffffff",
"mode": "search",
"page": "1",
"orderby": "create_datetime",
"text": "cute",
"stringtype": "and",
"keywords": "yes",
"title": "yes",
"description": "no",
},
},
})
def __init__(self, match): def __init__(self, match):
InkbunnyExtractor.__init__(self, match) InkbunnyExtractor.__init__(self, match)
@ -279,15 +202,8 @@ class InkbunnyFollowingExtractor(InkbunnyExtractor):
pattern = (BASE_PATTERN + r"/(?:" pattern = (BASE_PATTERN + r"/(?:"
r"watchlist_process\.php\?mode=watching&user_id=(\d+)|" r"watchlist_process\.php\?mode=watching&user_id=(\d+)|"
r"usersviewall\.php\?([^#]+&mode=watching&[^#]+))") r"usersviewall\.php\?([^#]+&mode=watching&[^#]+))")
test = ( example = ("https://inkbunny.net/watchlist_process.php"
(("https://inkbunny.net/watchlist_process.php" "?mode=watching&user_id=12345")
"?mode=watching&user_id=20969"), {
"pattern": InkbunnyUserExtractor.pattern,
"count": ">= 90",
}),
("https://inkbunny.net/usersviewall.php?rid=ffffffffff"
"&mode=watching&page=1&user_id=20969&orderby=added&namesonly="),
)
def __init__(self, match): def __init__(self, match):
InkbunnyExtractor.__init__(self, match) InkbunnyExtractor.__init__(self, match)
@ -324,16 +240,7 @@ class InkbunnyPostExtractor(InkbunnyExtractor):
"""Extractor for individual Inkbunny posts""" """Extractor for individual Inkbunny posts"""
subcategory = "post" subcategory = "post"
pattern = BASE_PATTERN + r"/s/(\d+)" pattern = BASE_PATTERN + r"/s/(\d+)"
test = ( example = "https://inkbunny.net/s/12345"
("https://inkbunny.net/s/1829715", {
"pattern": r"https://[\w.]+\.metapix\.net/files/full"
r"/2626/2626843_soina_dscn2296\.jpg",
"content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8",
}),
("https://inkbunny.net/s/2044094", {
"count": 4,
}),
)
def __init__(self, match): def __init__(self, match):
InkbunnyExtractor.__init__(self, match) InkbunnyExtractor.__init__(self, match)

164
gallery_dl/extractor/instagram.py
View File

@ -398,11 +398,7 @@ class InstagramUserExtractor(InstagramExtractor):
"""Extractor for an Instagram user profile""" """Extractor for an Instagram user profile"""
subcategory = "user" subcategory = "user"
pattern = USER_PATTERN + r"/?(?:$|[?#])" pattern = USER_PATTERN + r"/?(?:$|[?#])"
test = ( example = "https://www.instagram.com/USER/"
("https://www.instagram.com/instagram/"),
("https://www.instagram.com/instagram/?hl=en"),
("https://www.instagram.com/id:25025320/"),
)
def initialize(self): def initialize(self):
pass pass
@ -427,10 +423,7 @@ class InstagramPostsExtractor(InstagramExtractor):
"""Extractor for an Instagram user's posts""" """Extractor for an Instagram user's posts"""
subcategory = "posts" subcategory = "posts"
pattern = USER_PATTERN + r"/posts" pattern = USER_PATTERN + r"/posts"
test = ("https://www.instagram.com/instagram/posts/", { example = "https://www.instagram.com/USER/posts/"
"range": "1-16",
"count": ">= 16",
})
def posts(self): def posts(self):
uid = self.api.user_id(self.item) uid = self.api.user_id(self.item)
@ -441,10 +434,7 @@ class InstagramReelsExtractor(InstagramExtractor):
"""Extractor for an Instagram user's reels""" """Extractor for an Instagram user's reels"""
subcategory = "reels" subcategory = "reels"
pattern = USER_PATTERN + r"/reels" pattern = USER_PATTERN + r"/reels"
test = ("https://www.instagram.com/instagram/reels/", { example = "https://www.instagram.com/USER/reels/"
"range": "40-60",
"count": ">= 20",
})
def posts(self): def posts(self):
uid = self.api.user_id(self.item) uid = self.api.user_id(self.item)
@ -455,15 +445,7 @@ class InstagramTaggedExtractor(InstagramExtractor):
"""Extractor for an Instagram user's tagged posts""" """Extractor for an Instagram user's tagged posts"""
subcategory = "tagged" subcategory = "tagged"
pattern = USER_PATTERN + r"/tagged" pattern = USER_PATTERN + r"/tagged"
test = ("https://www.instagram.com/instagram/tagged/", { example = "https://www.instagram.com/USER/tagged/"
"range": "1-16",
"count": ">= 16",
"keyword": {
"tagged_owner_id" : "25025320",
"tagged_username" : "instagram",
"tagged_full_name": "Instagram",
},
})
def metadata(self): def metadata(self):
if self.item.startswith("id:"): if self.item.startswith("id:"):
@ -487,11 +469,7 @@ class InstagramGuideExtractor(InstagramExtractor):
"""Extractor for an Instagram guide""" """Extractor for an Instagram guide"""
subcategory = "guide" subcategory = "guide"
pattern = USER_PATTERN + r"/guide/[^/?#]+/(\d+)" pattern = USER_PATTERN + r"/guide/[^/?#]+/(\d+)"
test = (("https://www.instagram.com/kadakaofficial/guide" example = "https://www.instagram.com/USER/guide/"
"/knit-i-need-collection/18131821684305217/"), {
"range": "1-16",
"count": ">= 16",
})
def __init__(self, match): def __init__(self, match):
InstagramExtractor.__init__(self, match) InstagramExtractor.__init__(self, match)
@ -508,10 +486,7 @@ class InstagramSavedExtractor(InstagramExtractor):
"""Extractor for an Instagram user's saved media""" """Extractor for an Instagram user's saved media"""
subcategory = "saved" subcategory = "saved"
pattern = USER_PATTERN + r"/saved(?:/all-posts)?/?$" pattern = USER_PATTERN + r"/saved(?:/all-posts)?/?$"
test = ( example = "https://www.instagram.com/USER/saved/"
("https://www.instagram.com/instagram/saved/"),
("https://www.instagram.com/instagram/saved/all-posts/"),
)
def posts(self): def posts(self):
return self.api.user_saved() return self.api.user_saved()
@ -521,9 +496,7 @@ class InstagramCollectionExtractor(InstagramExtractor):
"""Extractor for Instagram collection""" """Extractor for Instagram collection"""
subcategory = "collection" subcategory = "collection"
pattern = USER_PATTERN + r"/saved/([^/?#]+)/([^/?#]+)" pattern = USER_PATTERN + r"/saved/([^/?#]+)/([^/?#]+)"
test = ( example = "https://www.instagram.com/USER/saved/COLLECTION/12345"
"https://www.instagram.com/instagram/saved/collection_name/123456789/",
)
def __init__(self, match): def __init__(self, match):
InstagramExtractor.__init__(self, match) InstagramExtractor.__init__(self, match)
@ -545,14 +518,7 @@ class InstagramStoriesExtractor(InstagramExtractor):
pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" pattern = (r"(?:https?://)?(?:www\.)?instagram\.com"
r"/s(?:tories/(?:highlights/(\d+)|([^/?#]+)(?:/(\d+))?)" r"/s(?:tories/(?:highlights/(\d+)|([^/?#]+)(?:/(\d+))?)"
r"|/(aGlnaGxpZ2h0[^?#]+)(?:\?story_media_id=(\d+))?)") r"|/(aGlnaGxpZ2h0[^?#]+)(?:\?story_media_id=(\d+))?)")
test = ( example = "https://www.instagram.com/stories/USER/"
("https://www.instagram.com/stories/instagram/"),
("https://www.instagram.com/stories/highlights/18042509488170095/"),
("https://instagram.com/stories/geekmig/2724343156064789461"),
("https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1"),
("https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1"
"?story_media_id=2724343156064789461"),
)
def __init__(self, match): def __init__(self, match):
h1, self.user, m1, h2, m2 = match.groups() h1, self.user, m1, h2, m2 = match.groups()
@ -587,7 +553,7 @@ class InstagramHighlightsExtractor(InstagramExtractor):
"""Extractor for an Instagram user's story highlights""" """Extractor for an Instagram user's story highlights"""
subcategory = "highlights" subcategory = "highlights"
pattern = USER_PATTERN + r"/highlights" pattern = USER_PATTERN + r"/highlights"
test = ("https://www.instagram.com/instagram/highlights",) example = "https://www.instagram.com/USER/highlights/"
def posts(self): def posts(self):
uid = self.api.user_id(self.item) uid = self.api.user_id(self.item)
@ -598,10 +564,7 @@ class InstagramFollowingExtractor(InstagramExtractor):
"""Extractor for an Instagram user's followed users""" """Extractor for an Instagram user's followed users"""
subcategory = "following" subcategory = "following"
pattern = USER_PATTERN + r"/following" pattern = USER_PATTERN + r"/following"
test = ("https://www.instagram.com/instagram/following", { example = "https://www.instagram.com/USER/following/"
"range": "1-16",
"count": ">= 16",
})
def items(self): def items(self):
uid = self.api.user_id(self.item) uid = self.api.user_id(self.item)
@ -616,10 +579,7 @@ class InstagramTagExtractor(InstagramExtractor):
subcategory = "tag" subcategory = "tag"
directory_fmt = ("{category}", "{subcategory}", "{tag}") directory_fmt = ("{category}", "{subcategory}", "{tag}")
pattern = BASE_PATTERN + r"/explore/tags/([^/?#]+)" pattern = BASE_PATTERN + r"/explore/tags/([^/?#]+)"
test = ("https://www.instagram.com/explore/tags/instagram/", { example = "https://www.instagram.com/explore/tags/TAG/"
"range": "1-16",
"count": ">= 16",
})
def metadata(self): def metadata(self):
return {"tag": text.unquote(self.item)} return {"tag": text.unquote(self.item)}
@ -632,10 +592,7 @@ class InstagramAvatarExtractor(InstagramExtractor):
"""Extractor for an Instagram user's avatar""" """Extractor for an Instagram user's avatar"""
subcategory = "avatar" subcategory = "avatar"
pattern = USER_PATTERN + r"/avatar" pattern = USER_PATTERN + r"/avatar"
test = ("https://www.instagram.com/instagram/avatar", { example = "https://www.instagram.com/USER/avatar/"
"pattern": r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19"
r"/281440578_1088265838702675_6233856337905829714_n\.jpg",
})
def posts(self): def posts(self):
if self._logged_in: if self._logged_in:
@ -675,102 +632,7 @@ class InstagramPostExtractor(InstagramExtractor):
subcategory = "post" subcategory = "post"
pattern = (r"(?:https?://)?(?:www\.)?instagram\.com" pattern = (r"(?:https?://)?(?:www\.)?instagram\.com"
r"/(?:[^/?#]+/)?(?:p|tv|reel)/([^/?#]+)") r"/(?:[^/?#]+/)?(?:p|tv|reel)/([^/?#]+)")
test = ( example = "https://www.instagram.com/p/abcdefg/"
# GraphImage
("https://www.instagram.com/p/BqvsDleB3lV/", {
"pattern": r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)"
r"/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35"
r"/44877605_725955034447492_3123079845831750529_n.jpg",
"keyword": {
"date": "dt:2018-11-29 01:04:04",
"description": str,
"height": int,
"likes": int,
"location_id": "214424288",
"location_slug": "hong-kong",
"location_url": "re:/explore/locations/214424288/hong-kong/",
"media_id": "1922949326347663701",
"shortcode": "BqvsDleB3lV",
"post_id": "1922949326347663701",
"post_shortcode": "BqvsDleB3lV",
"post_url": "https://www.instagram.com/p/BqvsDleB3lV/",
"tags": ["#WHPsquares"],
"typename": "GraphImage",
"username": "instagram",
"width": int,
}
}),
# GraphSidecar
("https://www.instagram.com/p/BoHk1haB5tM/", {
"count": 5,
"keyword": {
"sidecar_media_id": "1875629777499953996",
"sidecar_shortcode": "BoHk1haB5tM",
"post_id": "1875629777499953996",
"post_shortcode": "BoHk1haB5tM",
"post_url": "https://www.instagram.com/p/BoHk1haB5tM/",
"num": int,
"likes": int,
"username": "instagram",
}
}),
# GraphVideo
("https://www.instagram.com/p/Bqxp0VSBgJg/", {
"pattern": r"/46840863_726311431074534_7805566102611403091_n\.mp4",
"keyword": {
"date": "dt:2018-11-29 19:23:58",
"description": str,
"height": int,
"likes": int,
"media_id": "1923502432034620000",
"post_url": "https://www.instagram.com/p/Bqxp0VSBgJg/",
"shortcode": "Bqxp0VSBgJg",
"tags": ["#ASMR"],
"typename": "GraphVideo",
"username": "instagram",
"width": int,
}
}),
# GraphVideo (IGTV)
("https://www.instagram.com/tv/BkQjCfsBIzi/", {
"pattern": r"/10000000_597132547321814_702169244961988209_n\.mp4",
"keyword": {
"date": "dt:2018-06-20 19:51:32",
"description": str,
"height": int,
"likes": int,
"media_id": "1806097553666903266",
"post_url": "https://www.instagram.com/p/BkQjCfsBIzi/",
"shortcode": "BkQjCfsBIzi",
"typename": "GraphVideo",
"username": "instagram",
"width": int,
}
}),
# GraphSidecar with 2 embedded GraphVideo objects
("https://www.instagram.com/p/BtOvDOfhvRr/", {
"count": 2,
"keyword": {
"post_url": "https://www.instagram.com/p/BtOvDOfhvRr/",
"sidecar_media_id": "1967717017113261163",
"sidecar_shortcode": "BtOvDOfhvRr",
"video_url": str,
}
}),
# GraphImage with tagged user
("https://www.instagram.com/p/B_2lf3qAd3y/", {
"keyword": {
"tagged_users": [{
"id" : "1246468638",
"username" : "kaaymbl",
"full_name": "Call Me Kay",
}]
}
}),
# URL with username (#2085)
("https://www.instagram.com/dm/p/CW042g7B9CY/"),
("https://www.instagram.com/reel/CDg_6Y1pxWu/"),
)
def posts(self): def posts(self):
return self.api.media(self.item) return self.api.media(self.item)

31
gallery_dl/extractor/issuu.py
View File

@ -26,31 +26,7 @@ class IssuuPublicationExtractor(IssuuBase, GalleryExtractor):
filename_fmt = "{num:>03}.{extension}" filename_fmt = "{num:>03}.{extension}"
archive_fmt = "{document[publicationId]}_{num}" archive_fmt = "{document[publicationId]}_{num}"
pattern = r"(?:https?://)?issuu\.com(/[^/?#]+/docs/[^/?#]+)" pattern = r"(?:https?://)?issuu\.com(/[^/?#]+/docs/[^/?#]+)"
test = ("https://issuu.com/issuu/docs/motions-1-2019/", { example = "https://issuu.com/issuu/docs/TITLE/"
"pattern": r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg",
"count" : 36,
"keyword": {
"document": {
"access" : "PUBLIC",
"contentRating" : {
"isAdsafe" : True,
"isExplicit": False,
"isReviewed": True,
},
"date" : "dt:2019-09-16 00:00:00",
"description" : "re:Motions, the brand new publication by I",
"documentName" : "motions-1-2019",
"downloadable" : False,
"pageCount" : 36,
"publicationId" : "d99ec95935f15091b040cb8060f05510",
"title" : "Motions by Issuu - Issue 1",
"username" : "issuu",
},
"extension": "jpg",
"filename" : r"re:page_\d+",
"num" : int,
},
})
def metadata(self, page): def metadata(self, page):
data = util.json_loads(text.rextract( data = util.json_loads(text.rextract(
@ -78,10 +54,7 @@ class IssuuUserExtractor(IssuuBase, Extractor):
"""Extractor for all publications of a user/publisher""" """Extractor for all publications of a user/publisher"""
subcategory = "user" subcategory = "user"
pattern = r"(?:https?://)?issuu\.com/([^/?#]+)/?$" pattern = r"(?:https?://)?issuu\.com/([^/?#]+)/?$"
test = ("https://issuu.com/issuu", { example = "https://issuu.com/USER"
"pattern": IssuuPublicationExtractor.pattern,
"count" : "> 25",
})
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

66
gallery_dl/extractor/itaku.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2022 Mike Fährmann # Copyright 2022-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -63,12 +63,7 @@ class ItakuGalleryExtractor(ItakuExtractor):
"""Extractor for posts from an itaku user gallery""" """Extractor for posts from an itaku user gallery"""
subcategory = "gallery" subcategory = "gallery"
pattern = BASE_PATTERN + r"/profile/([^/?#]+)/gallery" pattern = BASE_PATTERN + r"/profile/([^/?#]+)/gallery"
test = ("https://itaku.ee/profile/piku/gallery", { example = "https://itaku.ee/profile/USER/gallery"
"pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs"
r"/[^/?#]+\.(jpg|png|gif)",
"range": "1-10",
"count": 10,
})
def posts(self): def posts(self):
return self.api.galleries_images(self.item) return self.api.galleries_images(self.item)
@ -77,62 +72,7 @@ class ItakuGalleryExtractor(ItakuExtractor):
class ItakuImageExtractor(ItakuExtractor): class ItakuImageExtractor(ItakuExtractor):
subcategory = "image" subcategory = "image"
pattern = BASE_PATTERN + r"/images/(\d+)" pattern = BASE_PATTERN + r"/images/(\d+)"
test = ( example = "https://itaku.ee/images/12345"
("https://itaku.ee/images/100471", {
"pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_imgs"
r"/220504_oUNIAFT\.png",
"count": 1,
"keyword": {
"already_pinned": None,
"blacklisted": {
"blacklisted_tags": [],
"is_blacklisted": False
},
"can_reshare": True,
"date": "dt:2022-05-05 19:21:17",
"date_added": "2022-05-05T19:21:17.674148Z",
"date_edited": "2022-05-25T14:37:46.220612Z",
"description": "sketch from drawpile",
"extension": "png",
"filename": "220504_oUNIAFT",
"hotness_score": float,
"id": 100471,
"image": "https://d1wmr8tlk3viaj.cloudfront.net/gallery_imgs"
"/220504_oUNIAFT.png",
"image_xl": "https://d1wmr8tlk3viaj.cloudfront.net"
"/gallery_imgs/220504_oUNIAFT/lg.jpg",
"liked_by_you": False,
"maturity_rating": "SFW",
"num_comments": int,
"num_likes": int,
"num_reshares": int,
"obj_tags": 136446,
"owner": 16775,
"owner_avatar": "https://d1wmr8tlk3viaj.cloudfront.net"
"/profile_pics/av2022r_vKYVywc/md.jpg",
"owner_displayname": "Piku",
"owner_username": "piku",
"reshared_by_you": False,
"sections": ["Fanart/Miku"],
"tags": list,
"tags_character": ["hatsune_miku"],
"tags_copyright": ["vocaloid"],
"tags_general" : ["twintails", "green_hair", "flag",
"gloves", "green_eyes", "female",
"racing_miku"],
"title": "Racing Miku 2022 Ver.",
"too_mature": False,
"uncompressed_filesize": "0.62",
"video": None,
"visibility": "PUBLIC",
},
}),
# video
("https://itaku.ee/images/19465", {
"pattern": r"https://d1wmr8tlk3viaj\.cloudfront\.net/gallery_vids"
r"/sleepy_af_OY5GHWw\.mp4",
}),
)
def posts(self): def posts(self):
return (self.api.image(self.item),) return (self.api.image(self.item),)

23
gallery_dl/extractor/itchio.py
View File

@ -21,28 +21,7 @@ class ItchioGameExtractor(Extractor):
filename_fmt = "{game[title]} ({id}).{extension}" filename_fmt = "{game[title]} ({id}).{extension}"
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = r"(?:https?://)?(\w+).itch\.io/([\w-]+)" pattern = r"(?:https?://)?(\w+).itch\.io/([\w-]+)"
test = ( example = "https://USER.itch.io/GAME"
("https://sirtartarus.itch.io/a-craft-of-mine", {
"pattern": r"https://\w+\.ssl\.hwcdn\.net/upload2"
r"/game/1983311/7723751\?",
"count": 1,
"keyword": {
"extension": "",
"filename": "7723751",
"game": {
"id": 1983311,
"noun": "game",
"title": "A Craft Of Mine",
"url": "https://sirtartarus.itch.io/a-craft-of-mine",
},
"user": {
"id": 4060052,
"name": "SirTartarus",
"url": "https://sirtartarus.itch.io",
},
},
}),
)
def __init__(self, match): def __init__(self, match):
self.user, self.slug = match.groups() self.user, self.slug = match.groups()

55
gallery_dl/extractor/jpgfish.py
View File

@ -35,29 +35,7 @@ class JpgfishImageExtractor(JpgfishExtractor):
"""Extractor for jpgfish Images""" """Extractor for jpgfish Images"""
subcategory = "image" subcategory = "image"
pattern = BASE_PATTERN + r"/img/((?:[^/?#]+\.)?(\w+))" pattern = BASE_PATTERN + r"/img/((?:[^/?#]+\.)?(\w+))"
test = ( example = "https://jpg1.su/img/TITLE.ID"
("https://jpg1.su/img/funnymeme.LecXGS", {
"pattern": r"https://simp3\.jpg\.church/images/funnymeme\.jpg",
"content": "098e5e9b17ad634358426e0ffd1c93871474d13c",
"keyword": {
"album": "",
"extension": "jpg",
"filename": "funnymeme",
"id": "LecXGS",
"url": "https://simp3.jpg.church/images/funnymeme.jpg",
"user": "exearco",
},
}),
("https://jpg.church/img/auCruA", {
"pattern": r"https://simp2\.jpg\.church/hannahowo_00457\.jpg",
"keyword": {"album": "401-500"},
}),
("https://jpeg.pet/img/funnymeme.LecXGS"),
("https://jpg.pet/img/funnymeme.LecXGS"),
("https://jpg.fishing/img/funnymeme.LecXGS"),
("https://jpg.fish/img/funnymeme.LecXGS"),
("https://jpg.church/img/funnymeme.LecXGS"),
)
def __init__(self, match): def __init__(self, match):
JpgfishExtractor.__init__(self, match) JpgfishExtractor.__init__(self, match)
@ -84,22 +62,7 @@ class JpgfishAlbumExtractor(JpgfishExtractor):
"""Extractor for jpgfish Albums""" """Extractor for jpgfish Albums"""
subcategory = "album" subcategory = "album"
pattern = BASE_PATTERN + r"/a(?:lbum)?/([^/?#]+)(/sub)?" pattern = BASE_PATTERN + r"/a(?:lbum)?/([^/?#]+)(/sub)?"
test = ( example = "https://jpg1.su/album/TITLE.ID"
("https://jpg1.su/album/CDilP/?sort=date_desc&page=1", {
"count": 2,
}),
("https://jpg.fishing/a/gunggingnsk.N9OOI", {
"count": 114,
}),
("https://jpg.fish/a/101-200.aNJ6A/", {
"count": 100,
}),
("https://jpg.church/a/hannahowo.aNTdH/sub", {
"count": 606,
}),
("https://jpeg.pet/album/CDilP/?sort=date_desc&page=1"),
("https://jpg.pet/album/CDilP/?sort=date_desc&page=1"),
)
def __init__(self, match): def __init__(self, match):
JpgfishExtractor.__init__(self, match) JpgfishExtractor.__init__(self, match)
@ -123,19 +86,7 @@ class JpgfishUserExtractor(JpgfishExtractor):
"""Extractor for jpgfish Users""" """Extractor for jpgfish Users"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/(?!img|a(?:lbum)?)([^/?#]+)(/albums)?" pattern = BASE_PATTERN + r"/(?!img|a(?:lbum)?)([^/?#]+)(/albums)?"
test = ( example = "https://jpg1.su/USER"
("https://jpg1.su/exearco", {
"count": 3,
}),
("https://jpg.church/exearco/albums", {
"count": 1,
}),
("https://jpeg.pet/exearco"),
("https://jpg.pet/exearco"),
("https://jpg.fishing/exearco"),
("https://jpg.fish/exearco"),
("https://jpg.church/exearco"),
)
def __init__(self, match): def __init__(self, match):
JpgfishExtractor.__init__(self, match) JpgfishExtractor.__init__(self, match)

17
gallery_dl/extractor/jschan.py
View File

@ -31,12 +31,7 @@ class JschanThreadExtractor(JschanExtractor):
filename_fmt = "{postId}{num:?-//} {filename}.{extension}" filename_fmt = "{postId}{num:?-//} {filename}.{extension}"
archive_fmt = "{board}_{postId}_{num}" archive_fmt = "{board}_{postId}_{num}"
pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)\.html" pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)\.html"
test = ( example = "https://94chan.org/a/thread/12345.html"
("https://94chan.org/art/thread/25.html", {
"pattern": r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?",
"count": ">= 15"
})
)
def __init__(self, match): def __init__(self, match):
JschanExtractor.__init__(self, match) JschanExtractor.__init__(self, match)
@ -71,15 +66,7 @@ class JschanBoardExtractor(JschanExtractor):
subcategory = "board" subcategory = "board"
pattern = (BASE_PATTERN + r"/([^/?#]+)" pattern = (BASE_PATTERN + r"/([^/?#]+)"
r"(?:/index\.html|/catalog\.html|/\d+\.html|/?$)") r"(?:/index\.html|/catalog\.html|/\d+\.html|/?$)")
test = ( example = "https://94chan.org/a/"
("https://94chan.org/art/", {
"pattern": JschanThreadExtractor.pattern,
"count": ">= 30"
}),
("https://94chan.org/art/2.html"),
("https://94chan.org/art/catalog.html"),
("https://94chan.org/art/index.html"),
)
def __init__(self, match): def __init__(self, match):
JschanExtractor.__init__(self, match) JschanExtractor.__init__(self, match)

13
gallery_dl/extractor/kabeuchi.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2020 Mike Fährmann # Copyright 2020-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -21,16 +21,7 @@ class KabeuchiUserExtractor(Extractor):
archive_fmt = "{id}_{num}" archive_fmt = "{id}_{num}"
root = "https://kabe-uchiroom.com" root = "https://kabe-uchiroom.com"
pattern = r"(?:https?://)?kabe-uchiroom\.com/mypage/?\?id=(\d+)" pattern = r"(?:https?://)?kabe-uchiroom\.com/mypage/?\?id=(\d+)"
test = ( example = "https://kabe-uchiroom.com/mypage/?id=12345"
("https://kabe-uchiroom.com/mypage/?id=919865303848255493", {
"pattern": (r"https://kabe-uchiroom\.com/accounts/upfile/3/"
r"919865303848255493/\w+\.jpe?g"),
"count": ">= 24",
}),
("https://kabe-uchiroom.com/mypage/?id=123456789", {
"exception": exception.NotFoundError,
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

28
gallery_dl/extractor/keenspot.py
View File

@ -20,33 +20,7 @@ class KeenspotComicExtractor(Extractor):
filename_fmt = "{filename}.{extension}" filename_fmt = "{filename}.{extension}"
archive_fmt = "{comic}_{filename}" archive_fmt = "{comic}_{filename}"
pattern = r"(?:https?://)?(?!www\.|forums\.)([\w-]+)\.keenspot\.com(/.+)?" pattern = r"(?:https?://)?(?!www\.|forums\.)([\w-]+)\.keenspot\.com(/.+)?"
test = ( example = "http://COMIC.keenspot.com/"
# link
("http://marksmen.keenspot.com/", {
"range": "1-3",
"url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6",
}),
# id
("http://barkercomic.keenspot.com/", {
"range": "1-3",
"url": "c4080926db18d00bac641fdd708393b7d61379e6",
}),
# id v2
("http://crowscare.keenspot.com/", {
"range": "1-3",
"url": "a00e66a133dd39005777317da90cef921466fcaa"
}),
# ks
("http://supernovas.keenspot.com/", {
"range": "1-3",
"url": "de21b12887ef31ff82edccbc09d112e3885c3aab"
}),
# "random" access
("http://twokinds.keenspot.com/comic/1066/", {
"range": "1-3",
"url": "6a784e11370abfb343dcad9adbb7718f9b7be350",
})
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

150
gallery_dl/extractor/kemonoparty.py
View File

@ -216,19 +216,7 @@ class KemonopartyUserExtractor(KemonopartyExtractor):
"""Extractor for all posts from a kemono.party user listing""" """Extractor for all posts from a kemono.party user listing"""
subcategory = "user" subcategory = "user"
pattern = USER_PATTERN + r"/?(?:\?o=(\d+))?(?:$|[?#])" pattern = USER_PATTERN + r"/?(?:\?o=(\d+))?(?:$|[?#])"
test = ( example = "https://kemono.party/SERVICE/user/12345"
("https://kemono.party/fanbox/user/6993449", {
"range": "1-25",
"count": 25,
}),
# 'max-posts' option, 'o' query parameter (#1674)
("https://kemono.party/patreon/user/881792?o=150", {
"options": (("max-posts", 25),),
"count": "< 100",
}),
("https://kemono.su/subscribestar/user/alcorart"),
("https://kemono.party/subscribestar/user/alcorart"),
)
def __init__(self, match): def __init__(self, match):
_, _, service, user_id, offset = match.groups() _, _, service, user_id, offset = match.groups()
@ -256,87 +244,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
"""Extractor for a single kemono.party post""" """Extractor for a single kemono.party post"""
subcategory = "post" subcategory = "post"
pattern = USER_PATTERN + r"/post/([^/?#]+)" pattern = USER_PATTERN + r"/post/([^/?#]+)"
test = ( example = "https://kemono.party/SERVICE/user/12345/post/12345"
("https://kemono.party/fanbox/user/6993449/post/506575", {
"pattern": r"https://kemono.party/data/21/0f"
r"/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg",
"content": "900949cefc97ab8dc1979cc3664785aac5ba70dd",
"keyword": {
"added": "Wed, 06 May 2020 20:28:02 GMT",
"content": str,
"count": 1,
"date": "dt:2019-08-11 02:09:04",
"edited": None,
"embed": dict,
"extension": "jpeg",
"filename": "P058kDFYus7DbqAkGlfWTlOr",
"hash": "210f35388e28bbcf756db18dd516e2d8"
"2ce758e0d32881eeee76d43e1716d382",
"id": "506575",
"num": 1,
"published": "Sun, 11 Aug 2019 02:09:04 GMT",
"service": "fanbox",
"shared_file": False,
"subcategory": "fanbox",
"title": "c96取り置き",
"type": "file",
"user": "6993449",
},
}),
# inline image (#1286)
("https://kemono.party/fanbox/user/7356311/post/802343", {
"pattern": r"https://kemono\.party/data/47/b5/47b5c014ecdcfabdf2c8"
r"5eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg",
"keyword": {"hash": "47b5c014ecdcfabdf2c85eec53f1133a"
"76336997ae8596f332e97d956a460ad2"},
}),
# kemono.party -> data.kemono.party
("https://kemono.party/gumroad/user/trylsc/post/IURjT", {
"pattern": r"https://kemono\.party/data/("
r"a4/7b/a47bfe938d8c1682eef06e885927484cd8df1b.+\.jpg|"
r"c6/04/c6048f5067fd9dbfa7a8be565ac194efdfb6e4.+\.zip)",
}),
# username (#1548, #1652)
("https://kemono.party/gumroad/user/3252870377455/post/aJnAH", {
"options": (("metadata", True),),
"keyword": {"username": "Kudalyn's Creations"},
}),
# skip patreon duplicates
("https://kemono.party/patreon/user/4158582/post/32099982", {
"count": 2,
}),
# allow duplicates (#2440)
("https://kemono.party/patreon/user/4158582/post/32099982", {
"options": (("duplicates", True),),
"count": 3,
}),
# DMs (#2008)
("https://kemono.party/patreon/user/34134344/post/38129255", {
"options": (("dms", True),),
"keyword": {"dms": [{
"body": r"re:Hi! Thank you very much for supporting the work I"
r" did in May. Here's your reward pack! I hope you fin"
r"d something you enjoy in it. :\)\n\nhttps://www.medi"
r"afire.com/file/\w+/Set13_tier_2.zip/file",
"date": "2021-07-31 02:47:51.327865",
}]},
}),
# coomer.party (#2100)
("https://coomer.party/onlyfans/user/alinity/post/125962203", {
"pattern": r"https://coomer\.party/data/7d/3f/7d3fd9804583dc224968"
r"c0591163ec91794552b04f00a6c2f42a15b68231d5a8\.jpg",
}),
# invalid file (#3510)
("https://kemono.party/patreon/user/19623797/post/29035449", {
"pattern": r"907ba78b4545338d3539683e63ecb51c"
r"f51c10adc9dabd86e92bd52339f298b9\.txt",
"content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", # empty
}),
("https://kemono.su/subscribestar/user/alcorart/post/184330"),
("https://kemono.party/subscribestar/user/alcorart/post/184330"),
("https://www.kemono.party/subscribestar/user/alcorart/post/184330"),
("https://beta.kemono.party/subscribestar/user/alcorart/post/184330"),
)
def __init__(self, match): def __init__(self, match):
_, _, service, user_id, post_id = match.groups() _, _, service, user_id, post_id = match.groups()
@ -359,30 +267,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
filename_fmt = "{id}_{num:>02}_{filename}.{extension}" filename_fmt = "{id}_{num:>02}_{filename}.{extension}"
archive_fmt = "discord_{server}_{id}_{num}" archive_fmt = "discord_{server}_{id}_{num}"
pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)" pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)"
test = ( example = "https://kemono.party/discard/server/12345/channel/12345"
(("https://kemono.party/discord"
"/server/488668827274444803#finish-work"), {
"count": 4,
"keyword": {"channel_name": "finish-work"},
}),
(("https://kemono.su/discord"
"/server/256559665620451329/channel/462437519519383555#"), {
"pattern": r"https://kemono\.su/data/("
r"e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|"
r"51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)",
"keyword": {"hash": "re:e377e3525164559484ace2e64425b0cec1db08"
"|51453640a5e0a4d23fbf57fb85390f9c5ec154"},
"count": ">= 2",
}),
# 'inline' files
(("https://kemono.party/discord"
"/server/315262215055736843/channel/315262215055736843#general"), {
"pattern": r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$",
"options": (("image-filter", "type == 'inline'"),),
"keyword": {"hash": ""},
"range": "1-5",
}),
)
def __init__(self, match): def __init__(self, match):
KemonopartyExtractor.__init__(self, match) KemonopartyExtractor.__init__(self, match)
@ -461,16 +346,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
class KemonopartyDiscordServerExtractor(KemonopartyExtractor): class KemonopartyDiscordServerExtractor(KemonopartyExtractor):
subcategory = "discord-server" subcategory = "discord-server"
pattern = BASE_PATTERN + r"/discord/server/(\d+)$" pattern = BASE_PATTERN + r"/discord/server/(\d+)$"
test = ( example = "https://kemono.party/discard/server/12345"
("https://kemono.party/discord/server/488668827274444803", {
"pattern": KemonopartyDiscordExtractor.pattern,
"count": 13,
}),
("https://kemono.su/discord/server/488668827274444803", {
"pattern": KemonopartyDiscordExtractor.pattern,
"count": 13,
}),
)
def __init__(self, match): def __init__(self, match):
KemonopartyExtractor.__init__(self, match) KemonopartyExtractor.__init__(self, match)
@ -492,23 +368,7 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor):
"""Extractor for kemono.party favorites""" """Extractor for kemono.party favorites"""
subcategory = "favorite" subcategory = "favorite"
pattern = BASE_PATTERN + r"/favorites(?:/?\?([^#]+))?" pattern = BASE_PATTERN + r"/favorites(?:/?\?([^#]+))?"
test = ( example = "https://kemono.party/favorites"
("https://kemono.party/favorites", {
"pattern": KemonopartyUserExtractor.pattern,
"url": "f4b5b796979bcba824af84206578c79101c7f0e1",
"count": 3,
}),
("https://kemono.party/favorites?type=post", {
"pattern": KemonopartyPostExtractor.pattern,
"url": "ecfccf5f0d50b8d14caa7bbdcf071de5c1e5b90f",
"count": 3,
}),
("https://kemono.su/favorites?type=post", {
"pattern": KemonopartyPostExtractor.pattern,
"url": "4be8e84cb384a907a8e7997baaf6287b451783b5",
"count": 3,
}),
)
def __init__(self, match): def __init__(self, match):
KemonopartyExtractor.__init__(self, match) KemonopartyExtractor.__init__(self, match)

25
gallery_dl/extractor/khinsider.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2016-2022 Mike Fährmann # Copyright 2016-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -16,30 +16,13 @@ class KhinsiderSoundtrackExtractor(AsynchronousMixin, Extractor):
"""Extractor for soundtracks from khinsider.com""" """Extractor for soundtracks from khinsider.com"""
category = "khinsider" category = "khinsider"
subcategory = "soundtrack" subcategory = "soundtrack"
root = "https://downloads.khinsider.com"
directory_fmt = ("{category}", "{album[name]}") directory_fmt = ("{category}", "{album[name]}")
archive_fmt = "{filename}.{extension}" archive_fmt = "{filename}.{extension}"
pattern = (r"(?:https?://)?downloads\.khinsider\.com" pattern = (r"(?:https?://)?downloads\.khinsider\.com"
r"/game-soundtracks/album/([^/?#]+)") r"/game-soundtracks/album/([^/?#]+)")
root = "https://downloads.khinsider.com" example = ("https://downloads.khinsider.com"
test = (("https://downloads.khinsider.com" "/game-soundtracks/album/TITLE")
"/game-soundtracks/album/horizon-riders-wii"), {
"pattern": r"https?://vgm(site|downloads)\.com"
r"/soundtracks/horizon-riders-wii/[^/]+"
r"/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3",
"keyword": {
"album": {
"count": 1,
"date": "Sep 18th, 2016",
"name": "Horizon Riders",
"platform": "Wii",
"size": 26214400,
"type": "Gamerip",
},
"extension": "mp3",
"filename": "Horizon Riders Wii - Full Soundtrack",
},
"count": 1,
})
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

23
gallery_dl/extractor/komikcast.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2018-2022 Mike Fährmann # Copyright 2018-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -48,18 +48,7 @@ class KomikcastBase():
class KomikcastChapterExtractor(KomikcastBase, ChapterExtractor): class KomikcastChapterExtractor(KomikcastBase, ChapterExtractor):
"""Extractor for manga-chapters from komikcast.site""" """Extractor for manga-chapters from komikcast.site"""
pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/)" pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/)"
test = ( example = "https://komikcast.site/chapter/TITLE"
(("https://komikcast.site/chapter"
"/apotheosis-chapter-02-2-bahasa-indonesia/"), {
"url": "f6b43fbc027697749b3ea1c14931c83f878d7936",
"keyword": "f3938e1aff9ad1f302f52447e9781b21f6da26d4",
}),
(("https://komikcast.me/chapter"
"/soul-land-ii-chapter-300-1-bahasa-indonesia/"), {
"url": "efd00a9bd95461272d51990d7bc54b79ff3ff2e6",
"keyword": "cb646cfed3d45105bd645ab38b2e9f7d8c436436",
}),
)
def metadata(self, page): def metadata(self, page):
info = text.extr(page, "<title>", " - Komikcast<") info = text.extr(page, "<title>", " - Komikcast<")
@ -79,13 +68,7 @@ class KomikcastMangaExtractor(KomikcastBase, MangaExtractor):
"""Extractor for manga from komikcast.site""" """Extractor for manga from komikcast.site"""
chapterclass = KomikcastChapterExtractor chapterclass = KomikcastChapterExtractor
pattern = BASE_PATTERN + r"(/(?:komik/)?[^/?#]+)/?$" pattern = BASE_PATTERN + r"(/(?:komik/)?[^/?#]+)/?$"
test = ( example = "https://komikcast.site/komik/TITLE"
("https://komikcast.site/komik/090-eko-to-issho/", {
"url": "19d3d50d532e84be6280a3d61ff0fd0ca04dd6b4",
"keyword": "837a7e96867344ff59d840771c04c20dc46c0ab1",
}),
("https://komikcast.me/tonari-no-kashiwagi-san/"),
)
def chapters(self, page): def chapters(self, page):
results = [] results = []

33
gallery_dl/extractor/lensdump.py
View File

@ -48,19 +48,7 @@ class LensdumpBase():
class LensdumpAlbumExtractor(LensdumpBase, GalleryExtractor): class LensdumpAlbumExtractor(LensdumpBase, GalleryExtractor):
subcategory = "album" subcategory = "album"
pattern = BASE_PATTERN + r"/(?:((?!\w+/albums|a/|i/)\w+)|a/(\w+))" pattern = BASE_PATTERN + r"/(?:((?!\w+/albums|a/|i/)\w+)|a/(\w+))"
test = ( example = "https://lensdump.com/a/ID"
("https://lensdump.com/a/1IhJr", {
"pattern": r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png",
"keyword": {
"extension": "png",
"name": str,
"num": int,
"title": str,
"url": str,
"width": int,
},
}),
)
def __init__(self, match): def __init__(self, match):
GalleryExtractor.__init__(self, match, match.string) GalleryExtractor.__init__(self, match, match.string)
@ -100,7 +88,7 @@ class LensdumpAlbumsExtractor(LensdumpBase, Extractor):
"""Extractor for album list from lensdump.com""" """Extractor for album list from lensdump.com"""
subcategory = "albums" subcategory = "albums"
pattern = BASE_PATTERN + r"/\w+/albums" pattern = BASE_PATTERN + r"/\w+/albums"
test = ("https://lensdump.com/vstar925/albums",) example = "https://lensdump.com/USER/albums"
def items(self): def items(self):
for node in self.nodes(): for node in self.nodes():
@ -117,22 +105,7 @@ class LensdumpImageExtractor(LensdumpBase, Extractor):
directory_fmt = ("{category}",) directory_fmt = ("{category}",)
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = BASE_PATTERN + r"/i/(\w+)" pattern = BASE_PATTERN + r"/i/(\w+)"
test = ( example = "https://lensdump.com/i/ID"
("https://lensdump.com/i/tyoAyM", {
"pattern": r"https://c\.l3n\.co/i/tyoAyM\.webp",
"content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46",
"keyword": {
"date": "dt:2022-08-01 08:24:28",
"extension": "webp",
"filename": "tyoAyM",
"height": 400,
"id": "tyoAyM",
"title": "MYOBI clovis bookcaseset",
"url": "https://c.l3n.co/i/tyoAyM.webp",
"width": 620,
},
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

32
gallery_dl/extractor/lexica.py
View File

@ -20,37 +20,7 @@ class LexicaSearchExtractor(Extractor):
directory_fmt = ("{category}", "{search_tags}") directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = r"(?:https?://)?lexica\.art/?\?q=([^&#]+)" pattern = r"(?:https?://)?lexica\.art/?\?q=([^&#]+)"
test = ( example = "https://lexica.art/?q=QUERY"
("https://lexica.art/?q=tree", {
"pattern": r"https://lexica-serve-encoded-images2\.sharif\."
r"workers.dev/full_jpg/[0-9a-f-]{36}$",
"range": "1-80",
"count": 80,
"keyword": {
"height": int,
"id": str,
"upscaled_height": int,
"upscaled_width": int,
"userid": str,
"width": int,
"prompt": {
"c": int,
"grid": bool,
"height": int,
"id": str,
"images": list,
"initImage": None,
"initImageStrength": None,
"model": "lexica-aperture-v2",
"negativePrompt": str,
"prompt": str,
"seed": str,
"timestamp": r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ",
"width": int,
},
},
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

19
gallery_dl/extractor/lightroom.py
View File

@ -18,24 +18,7 @@ class LightroomGalleryExtractor(Extractor):
filename_fmt = "{num:>04}_{id}.{extension}" filename_fmt = "{num:>04}_{id}.{extension}"
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = r"(?:https?://)?lightroom\.adobe\.com/shares/([0-9a-f]+)" pattern = r"(?:https?://)?lightroom\.adobe\.com/shares/([0-9a-f]+)"
test = ( example = "https://lightroom.adobe.com/shares/0123456789abcdef"
(("https://lightroom.adobe.com/shares/"
"0c9cce2033f24d24975423fe616368bf"), {
"keyword": {
"title": "Sterne und Nachtphotos",
"user": "Christian Schrang",
},
"count": ">= 55",
}),
(("https://lightroom.adobe.com/shares/"
"7ba68ad5a97e48608d2e6c57e6082813"), {
"keyword": {
"title": "HEBFC Snr/Res v Brighton",
"user": "",
},
"count": ">= 180",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

44
gallery_dl/extractor/livedoor.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2019-2020 Mike Fährmann # Copyright 2019-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -84,32 +84,7 @@ class LivedoorBlogExtractor(LivedoorExtractor):
"""Extractor for a user's blog on blog.livedoor.jp""" """Extractor for a user's blog on blog.livedoor.jp"""
subcategory = "blog" subcategory = "blog"
pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/?(?:$|[?#])" pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/?(?:$|[?#])"
test = ( example = "http://blog.livedoor.jp/USER/"
("http://blog.livedoor.jp/zatsu_ke/", {
"range": "1-50",
"count": 50,
"archive": False,
"pattern": r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+",
"keyword": {
"post": {
"categories" : tuple,
"date" : "type:datetime",
"description": str,
"id" : int,
"tags" : list,
"title" : str,
"user" : "zatsu_ke"
},
"filename": str,
"hash" : r"re:\w{4,}",
"num" : int,
},
}),
("http://blog.livedoor.jp/uotapo/", {
"range": "1-5",
"count": 5,
}),
)
def posts(self): def posts(self):
url = "{}/{}".format(self.root, self.user) url = "{}/{}".format(self.root, self.user)
@ -129,20 +104,7 @@ class LivedoorPostExtractor(LivedoorExtractor):
"""Extractor for images from a blog post on blog.livedoor.jp""" """Extractor for images from a blog post on blog.livedoor.jp"""
subcategory = "post" subcategory = "post"
pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/archives/(\d+)" pattern = r"(?:https?://)?blog\.livedoor\.jp/(\w+)/archives/(\d+)"
test = ( example = "http://blog.livedoor.jp/USER/archives/12345.html"
("http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", {
"url": "9ca3bbba62722c8155be79ad7fc47be409e4a7a2",
"keyword": "1f5b558492e0734f638b760f70bfc0b65c5a97b9",
}),
("http://blog.livedoor.jp/amaumauma/archives/7835811.html", {
"url": "204bbd6a9db4969c50e0923855aeede04f2e4a62",
"keyword": "05821c7141360e6057ef2d382b046f28326a799d",
}),
("http://blog.livedoor.jp/uotapo/archives/1050616939.html", {
"url": "4b5ab144b7309eb870d9c08f8853d1abee9946d2",
"keyword": "84fbf6e4eef16675013d6333039a7cfcb22c2d50",
}),
)
def __init__(self, match): def __init__(self, match):
LivedoorExtractor.__init__(self, match) LivedoorExtractor.__init__(self, match)

14
gallery_dl/extractor/lolisafe.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2021-2022 Mike Fährmann # Copyright 2021-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -30,17 +30,7 @@ BASE_PATTERN = LolisafeExtractor.update({
class LolisafeAlbumExtractor(LolisafeExtractor): class LolisafeAlbumExtractor(LolisafeExtractor):
subcategory = "album" subcategory = "album"
pattern = BASE_PATTERN + "/a/([^/?#]+)" pattern = BASE_PATTERN + "/a/([^/?#]+)"
test = ( example = "https://xbunkr.com/a/ID"
("https://xbunkr.com/a/TA0bu3F4", {
"pattern": r"https://media\.xbunkr\.com/[^.]+\.\w+",
"count": 861,
"keyword": {
"album_id": "TA0bu3F4",
"album_name": "Hannahowo Onlyfans Photos",
}
}),
("https://xbunkr.com/a/GNQc2I5d"),
)
def __init__(self, match): def __init__(self, match):
LolisafeExtractor.__init__(self, match) LolisafeExtractor.__init__(self, match)

78
gallery_dl/extractor/luscious.py
View File

@ -47,73 +47,7 @@ class LusciousAlbumExtractor(LusciousExtractor):
archive_fmt = "{album[id]}_{id}" archive_fmt = "{album[id]}_{id}"
pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net" pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
r"/(?:albums|pictures/c/[^/?#]+/album)/[^/?#]+_(\d+)") r"/(?:albums|pictures/c/[^/?#]+/album)/[^/?#]+_(\d+)")
test = ( example = "https://luscious.net/albums/TITLE_12345/"
("https://luscious.net/albums/okinami-no-koigokoro_277031/", {
"pattern": r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+"
r"/images/NTRshouldbeillegal/277031"
r"/luscious_net_\d+_\d+\.jpg$",
# "content": "b3a747a6464509440bd0ff6d1267e6959f8d6ff3",
"keyword": {
"album": {
"__typename" : "Album",
"audiences" : list,
"content" : "Hentai",
"cover" : "re:https://\\w+.luscious.net/.+/277031/",
"created" : 1479625853,
"created_by" : "NTRshouldbeillegal",
"date" : "dt:2016-11-20 07:10:53",
"description" : "Enjoy.",
"download_url": "re:/download/(r/)?824778/277031/",
"genres" : list,
"id" : 277031,
"is_manga" : True,
"labels" : list,
"language" : "English",
"like_status" : "none",
"modified" : int,
"permissions" : list,
"rating" : float,
"slug" : "okinami-no-koigokoro",
"status" : None,
"tags" : list,
"title" : "Okinami no Koigokoro",
"url" : "/albums/okinami-no-koigokoro_277031/",
"marked_for_deletion": False,
"marked_for_processing": False,
"number_of_animated_pictures": 0,
"number_of_favorites": int,
"number_of_pictures": 18,
},
"aspect_ratio": r"re:\d+:\d+",
"category" : "luscious",
"created" : int,
"date" : "type:datetime",
"height" : int,
"id" : int,
"is_animated" : False,
"like_status" : "none",
"position" : int,
"resolution" : r"re:\d+x\d+",
"status" : None,
"tags" : list,
"thumbnail" : str,
"title" : str,
"width" : int,
"number_of_comments": int,
"number_of_favorites": int,
},
}),
("https://luscious.net/albums/not-found_277035/", {
"exception": exception.NotFoundError,
}),
("https://members.luscious.net/albums/login-required_323871/", {
"count": 64,
}),
("https://www.luscious.net/albums/okinami_277031/"),
("https://members.luscious.net/albums/okinami_277031/"),
("https://luscious.net/pictures/c/video_game_manga/album"
"/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1"),
)
def __init__(self, match): def __init__(self, match):
LusciousExtractor.__init__(self, match) LusciousExtractor.__init__(self, match)
@ -338,15 +272,7 @@ class LusciousSearchExtractor(LusciousExtractor):
subcategory = "search" subcategory = "search"
pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net" pattern = (r"(?:https?://)?(?:www\.|members\.)?luscious\.net"
r"/albums/list/?(?:\?([^#]+))?") r"/albums/list/?(?:\?([^#]+))?")
test = ( example = "https://luscious.net/albums/list/?tagged=TAG"
("https://members.luscious.net/albums/list/"),
("https://members.luscious.net/albums/list/"
"?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", {
"pattern": LusciousAlbumExtractor.pattern,
"range": "41-60",
"count": 20,
}),
)
def __init__(self, match): def __init__(self, match):
LusciousExtractor.__init__(self, match) LusciousExtractor.__init__(self, match)

36
gallery_dl/extractor/lynxchan.py
View File

@ -40,22 +40,7 @@ class LynxchanThreadExtractor(LynxchanExtractor):
filename_fmt = "{postId}{num:?-//} {filename}.{extension}" filename_fmt = "{postId}{num:?-//} {filename}.{extension}"
archive_fmt = "{boardUri}_{postId}_{num}" archive_fmt = "{boardUri}_{postId}_{num}"
pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)"
test = ( example = "https://bbw-chan.nl/a/res/12345.html"
("https://bbw-chan.nl/bbwdraw/res/499.html", {
"pattern": r"https://bbw-chan\.nl/\.media/[0-9a-f]{64}(\.\w+)?$",
"count": ">= 352",
}),
("https://bbw-chan.nl/bbwdraw/res/489.html"),
("https://kohlchan.net/a/res/4594.html", {
"pattern": r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$",
"count": ">= 80",
}),
("https://endchan.org/yuri/res/193483.html", {
"pattern": r"https://endchan\.org/\.media/[^.]+(\.\w+)?$",
"count" : ">= 19",
}),
("https://endchan.org/yuri/res/33621.html"),
)
def __init__(self, match): def __init__(self, match):
LynxchanExtractor.__init__(self, match) LynxchanExtractor.__init__(self, match)
@ -86,24 +71,7 @@ class LynxchanBoardExtractor(LynxchanExtractor):
"""Extractor for LynxChan boards""" """Extractor for LynxChan boards"""
subcategory = "board" subcategory = "board"
pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)"
test = ( example = "https://bbw-chan.nl/a/"
("https://bbw-chan.nl/bbwdraw/", {
"pattern": LynxchanThreadExtractor.pattern,
"count": ">= 148",
}),
("https://bbw-chan.nl/bbwdraw/2.html"),
("https://kohlchan.net/a/", {
"pattern": LynxchanThreadExtractor.pattern,
"count": ">= 100",
}),
("https://kohlchan.net/a/2.html"),
("https://kohlchan.net/a/catalog.html"),
("https://endchan.org/yuri/", {
"pattern": LynxchanThreadExtractor.pattern,
"count" : ">= 9",
}),
("https://endchan.org/yuri/catalog.html"),
)
def __init__(self, match): def __init__(self, match):
LynxchanExtractor.__init__(self, match) LynxchanExtractor.__init__(self, match)

67
gallery_dl/extractor/mangadex.py
View File

@ -98,25 +98,8 @@ class MangadexChapterExtractor(MangadexExtractor):
"""Extractor for manga-chapters from mangadex.org""" """Extractor for manga-chapters from mangadex.org"""
subcategory = "chapter" subcategory = "chapter"
pattern = BASE_PATTERN + r"/chapter/([0-9a-f-]+)" pattern = BASE_PATTERN + r"/chapter/([0-9a-f-]+)"
test = ( example = ("https://mangadex.org/chapter"
("https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", { "/01234567-89ab-cdef-0123-456789abcdef")
"keyword": "e86128a79ebe7201b648f1caa828496a2878dc8f",
# "content": "50383a4c15124682057b197d40261641a98db514",
}),
# oneshot
("https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", {
"count": 64,
"keyword": "d11ed057a919854696853362be35fc0ba7dded4c",
}),
# MANGA Plus (#1154)
("https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", {
"exception": exception.StopExtraction,
}),
# 'externalUrl', but still downloadable / 404 (#2503)
("https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", {
"count": 0,
}),
)
def items(self): def items(self):
try: try:
@ -148,48 +131,8 @@ class MangadexMangaExtractor(MangadexExtractor):
"""Extractor for manga from mangadex.org""" """Extractor for manga from mangadex.org"""
subcategory = "manga" subcategory = "manga"
pattern = BASE_PATTERN + r"/(?:title|manga)/(?!feed$)([0-9a-f-]+)" pattern = BASE_PATTERN + r"/(?:title|manga)/(?!feed$)([0-9a-f-]+)"
test = ( example = ("https://mangadex.org/title"
("https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", { "/01234567-89ab-cdef-0123-456789abcdef")
"count": ">= 5",
"keyword": {
"manga" : "Souten no Koumori",
"manga_id": "f90c4398-8aad-4f51-8a1f-024ca09fdcbc",
"title" : "re:One[Ss]hot",
"volume" : 0,
"chapter" : 0,
"chapter_minor": "",
"chapter_id": str,
"date" : "type:datetime",
"lang" : str,
"language": str,
"artist" : ["Arakawa Hiromu"],
"author" : ["Arakawa Hiromu"],
"status" : "completed",
"tags" : ["Oneshot", "Historical", "Action",
"Martial Arts", "Drama", "Tragedy"],
},
}),
# mutliple values for 'lang' (#4093)
("https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", {
"options": (("lang", "fr,it"),),
"count": 2,
"keyword": {
"manga" : "Souten no Koumori",
"lang" : "re:fr|it",
"language": "re:French|Italian",
},
}),
("https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", {
"options": (("lang", "en"),),
"count": ">= 100",
}),
("https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", {
"count": 1,
}),
("https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", {
"count": ">= 20",
})
)
def chapters(self): def chapters(self):
return self.api.manga_feed(self.uuid) return self.api.manga_feed(self.uuid)
@ -199,7 +142,7 @@ class MangadexFeedExtractor(MangadexExtractor):
"""Extractor for chapters from your Followed Feed""" """Extractor for chapters from your Followed Feed"""
subcategory = "feed" subcategory = "feed"
pattern = BASE_PATTERN + r"/title/feed$()" pattern = BASE_PATTERN + r"/title/feed$()"
test = ("https://mangadex.org/title/feed",) example = "https://mangadex.org/title/feed"
def chapters(self): def chapters(self):
return self.api.user_follows_manga_feed() return self.api.user_follows_manga_feed()

40
gallery_dl/extractor/mangafox.py
View File

@ -20,14 +20,7 @@ class MangafoxChapterExtractor(ChapterExtractor):
root = "https://m.fanfox.net" root = "https://m.fanfox.net"
pattern = BASE_PATTERN + \ pattern = BASE_PATTERN + \
r"(/manga/[^/?#]+/((?:v([^/?#]+)/)?c(\d+)([^/?#]*)))" r"(/manga/[^/?#]+/((?:v([^/?#]+)/)?c(\d+)([^/?#]*)))"
test = ( example = "https://fanfox.net/manga/TITLE/v01/c001/1.html"
("http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", {
"keyword": "5661dab258d42d09d98f194f7172fb9851a49766",
"content": "5c50c252dcf12ffecf68801f4db8a2167265f66c",
}),
("http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/"),
("http://fanfox.net/manga/black_clover/vTBD/c295/1.html"),
)
def __init__(self, match): def __init__(self, match):
base, self.cstr, self.volume, self.chapter, self.minor = match.groups() base, self.cstr, self.volume, self.chapter, self.minor = match.groups()
@ -73,36 +66,7 @@ class MangafoxMangaExtractor(MangaExtractor):
root = "https://m.fanfox.net" root = "https://m.fanfox.net"
chapterclass = MangafoxChapterExtractor chapterclass = MangafoxChapterExtractor
pattern = BASE_PATTERN + r"(/manga/[^/?#]+)/?$" pattern = BASE_PATTERN + r"(/manga/[^/?#]+)/?$"
test = ( example = "https://fanfox.net/manga/TITLE"
("https://fanfox.net/manga/kanojo_mo_kanojo", {
"pattern": MangafoxChapterExtractor.pattern,
"count": ">=60",
"keyword": {
"author": "HIROYUKI",
"chapter": int,
"chapter_minor": r"re:^(\.\d+)?$",
"chapter_string": r"re:(v\d+/)?c\d+",
"date": "type:datetime",
"description": "High school boy Naoya gets a confession from M"
"omi, a cute and friendly girl. However, Naoya "
"already has a girlfriend, Seki... but Momi is "
"too good a catch to let go. Momi and Nagoya's "
"goal becomes clear: convince Seki to accept be"
"ing an item with the two of them. Will she bud"
"ge?",
"lang": "en",
"language": "English",
"manga": "Kanojo mo Kanojo",
"tags": ["Comedy", "Romance", "School Life", "Shounen"],
"volume": int,
},
}),
("https://mangafox.me/manga/shangri_la_frontier", {
"pattern": MangafoxChapterExtractor.pattern,
"count": ">=45",
}),
("https://m.fanfox.net/manga/sentai_daishikkaku"),
)
def chapters(self, page): def chapters(self, page):
results = [] results = []

32
gallery_dl/extractor/mangahere.py
View File

@ -25,18 +25,7 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor):
"""Extractor for manga-chapters from mangahere.cc""" """Extractor for manga-chapters from mangahere.cc"""
pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]/manga/" pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]/manga/"
r"([^/]+(?:/v0*(\d+))?/c([^/?#]+))") r"([^/]+(?:/v0*(\d+))?/c([^/?#]+))")
test = ( example = "https://www.mangahere.cc/manga/TITLE/c001/1.html"
("https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", {
"keyword": "7c98d7b50a47e6757b089aa875a53aa970cac66f",
"content": "708d475f06893b88549cbd30df1e3f9428f2c884",
}),
# URLs without HTTP scheme (#1070)
("https://www.mangahere.cc/manga/beastars/c196/1.html", {
"pattern": "https://zjcdn.mangahere.org/.*",
}),
("http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/"),
("http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/"),
)
def __init__(self, match): def __init__(self, match):
self.part, self.volume, self.chapter = match.groups() self.part, self.volume, self.chapter = match.groups()
@ -95,24 +84,7 @@ class MangahereMangaExtractor(MangahereBase, MangaExtractor):
chapterclass = MangahereChapterExtractor chapterclass = MangahereChapterExtractor
pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]" pattern = (r"(?:https?://)?(?:www\.|m\.)?mangahere\.c[co]"
r"(/manga/[^/?#]+/?)(?:#.*)?$") r"(/manga/[^/?#]+/?)(?:#.*)?$")
test = ( example = "https://www.mangahere.cc/manga/TITLE"
("https://www.mangahere.cc/manga/aria/", {
"url": "9c2e54ec42e9a87ad53096c328b33c90750af3e4",
"keyword": "71503c682c5d0c277a50409a8c5fd78e871e3d69",
"count": 71,
}),
("https://www.mangahere.cc/manga/hiyokoi/#50", {
"url": "654850570aa03825cd57e2ae2904af489602c523",
"keyword": "c8084d89a9ea6cf40353093669f9601a39bf5ca2",
}),
# adult filter (#556)
("http://www.mangahere.cc/manga/gunnm_mars_chronicle/", {
"pattern": MangahereChapterExtractor.pattern,
"count": ">= 50",
}),
("https://www.mangahere.co/manga/aria/"),
("https://m.mangahere.co/manga/aria/"),
)
def _init(self): def _init(self):
self.cookies.set("isAdult", "1", domain="www.mangahere.cc") self.cookies.set("isAdult", "1", domain="www.mangahere.cc")

19
gallery_dl/extractor/mangakakalot.py
View File

@ -25,16 +25,7 @@ class MangakakalotBase():
class MangakakalotChapterExtractor(MangakakalotBase, ChapterExtractor): class MangakakalotChapterExtractor(MangakakalotBase, ChapterExtractor):
"""Extractor for manga chapters from mangakakalot.tv""" """Extractor for manga chapters from mangakakalot.tv"""
pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/chapter[_-][^/?#]+)" pattern = BASE_PATTERN + r"(/chapter/[^/?#]+/chapter[_-][^/?#]+)"
test = ( example = "https://ww6.mangakakalot.tv/chapter/manga-ID/chapter-01"
("https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", {
"pattern": r"https://cm\.blazefast\.co"
r"/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg",
"keyword": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633",
"count": 9,
}),
("https://mangakakalot.tv/chapter"
"/hatarakanai_futari_the_jobless_siblings/chapter_20.1"),
)
def __init__(self, match): def __init__(self, match):
self.path = match.group(1) self.path = match.group(1)
@ -78,13 +69,7 @@ class MangakakalotMangaExtractor(MangakakalotBase, MangaExtractor):
"""Extractor for manga from mangakakalot.tv""" """Extractor for manga from mangakakalot.tv"""
chapterclass = MangakakalotChapterExtractor chapterclass = MangakakalotChapterExtractor
pattern = BASE_PATTERN + r"(/manga/[^/?#]+)" pattern = BASE_PATTERN + r"(/manga/[^/?#]+)"
test = ( example = "https://ww6.mangakakalot.tv/manga/manga-ID"
("https://ww3.mangakakalot.tv/manga/manga-jk986845", {
"pattern": MangakakalotChapterExtractor.pattern,
"count": ">= 30",
}),
("https://mangakakalot.tv/manga/lk921810"),
)
def chapters(self, page): def chapters(self, page):
data = {"lang": "en", "language": "English"} data = {"lang": "en", "language": "English"}

36
gallery_dl/extractor/manganelo.py
View File

@ -55,27 +55,7 @@ class ManganeloBase():
class ManganeloChapterExtractor(ManganeloBase, ChapterExtractor): class ManganeloChapterExtractor(ManganeloBase, ChapterExtractor):
"""Extractor for manga chapters from manganelo.com""" """Extractor for manga chapters from manganelo.com"""
pattern = BASE_PATTERN + r"(/(?:manga-\w+|chapter/\w+)/chapter[-_][^/?#]+)" pattern = BASE_PATTERN + r"(/(?:manga-\w+|chapter/\w+)/chapter[-_][^/?#]+)"
test = ( example = "https://chapmanganato.com/manga-ID/chapter-01"
("https://chapmanganato.com/manga-gn983696/chapter-23", {
"pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23"
r"/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg",
"keyword": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68",
"count": 25,
}),
("https://chapmanganelo.com/manga-ti107776/chapter-4", {
"pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92"
r"/08/ti970565/chapter_4_caster/\d+-o\.jpg",
"keyword": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded",
"count": 45,
}),
("https://chapmanganato.com/manga-no991297/chapter-8", {
"keyword": {"chapter": 8, "chapter_minor": "-1"},
"count": 20,
}),
("https://readmanganato.com/manga-gn983696/chapter-23"),
("https://manganelo.com/chapter/gamers/chapter_15"),
("https://manganelo.com/chapter/gq921227/chapter_23"),
)
def metadata(self, page): def metadata(self, page):
extr = text.extract_from(page) extr = text.extract_from(page)
@ -104,19 +84,7 @@ class ManganeloMangaExtractor(ManganeloBase, MangaExtractor):
"""Extractor for manga from manganelo.com""" """Extractor for manga from manganelo.com"""
chapterclass = ManganeloChapterExtractor chapterclass = ManganeloChapterExtractor
pattern = BASE_PATTERN + r"(/(?:manga[-/]|read_)\w+)/?$" pattern = BASE_PATTERN + r"(/(?:manga[-/]|read_)\w+)/?$"
test = ( example = "https://manganato.com/manga-ID"
("https://chapmanganato.com/manga-gn983696", {
"pattern": ManganeloChapterExtractor.pattern,
"count": ">= 25",
}),
("https://m.manganelo.com/manga-ti107776", {
"pattern": ManganeloChapterExtractor.pattern,
"count": ">= 12",
}),
("https://readmanganato.com/manga-gn983696"),
("https://manganelo.com/manga/read_otome_no_teikoku"),
("https://manganelo.com/manga/ol921234/"),
)
def chapters(self, page): def chapters(self, page):
results = [] results = []

70
gallery_dl/extractor/mangapark.py
View File

@ -35,39 +35,7 @@ class MangaparkBase():
class MangaparkChapterExtractor(MangaparkBase, ChapterExtractor): class MangaparkChapterExtractor(MangaparkBase, ChapterExtractor):
"""Extractor for manga-chapters from mangapark.net""" """Extractor for manga-chapters from mangapark.net"""
pattern = BASE_PATTERN + r"/title/[^/?#]+/(\d+)" pattern = BASE_PATTERN + r"/title/[^/?#]+/(\d+)"
test = ( example = "https://mangapark.net/title/MANGA/12345-en-ch.01"
("https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", {
"count": 70,
"pattern": r"https://[\w-]+\.mpcdn\.org/comic/2002/e67"
r"/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg"
r"\?acc=[^&#]+&exp=\d+",
"keyword": {
"artist": [],
"author": ["Amano Kozue"],
"chapter": 60,
"chapter_id": 6710214,
"chapter_minor": ".2",
"count": 70,
"date": "dt:2022-01-15 09:25:03",
"extension": "jpeg",
"filename": str,
"genre": ["adventure", "comedy", "drama", "sci_fi",
"shounen", "slice_of_life"],
"lang": "en",
"language": "English",
"manga": "Aria",
"manga_id": 114972,
"page": int,
"source": "Koala",
"title": "Special Navigation - Aquaria Ii",
"volume": 12,
},
}),
("https://mangapark.com/title/114972-aria/6710214-en-ch.60.2"),
("https://mangapark.org/title/114972-aria/6710214-en-ch.60.2"),
("https://mangapark.io/title/114972-aria/6710214-en-ch.60.2"),
("https://mangapark.me/title/114972-aria/6710214-en-ch.60.2"),
)
def __init__(self, match): def __init__(self, match):
self.root = text.root_from_url(match.group(0)) self.root = text.root_from_url(match.group(0))
@ -115,41 +83,7 @@ class MangaparkMangaExtractor(MangaparkBase, Extractor):
"""Extractor for manga from mangapark.net""" """Extractor for manga from mangapark.net"""
subcategory = "manga" subcategory = "manga"
pattern = BASE_PATTERN + r"/title/(\d+)(?:-[^/?#]*)?/?$" pattern = BASE_PATTERN + r"/title/(\d+)(?:-[^/?#]*)?/?$"
test = ( example = "https://mangapark.net/title/MANGA"
("https://mangapark.net/title/114972-aria", {
"count": 141,
"pattern": MangaparkChapterExtractor.pattern,
"keyword": {
"chapter": int,
"chapter_id": int,
"chapter_minor": str,
"date": "type:datetime",
"lang": "en",
"language": "English",
"manga_id": 114972,
"source": "re:Horse|Koala",
"source_id": int,
"title": str,
"volume": int,
},
}),
# 'source' option
("https://mangapark.net/title/114972-aria", {
"options": (("source", "koala"),),
"count": 70,
"pattern": MangaparkChapterExtractor.pattern,
"keyword": {
"source": "Koala",
"source_id": 15150116,
},
}),
("https://mangapark.com/title/114972-"),
("https://mangapark.com/title/114972"),
("https://mangapark.com/title/114972-aria"),
("https://mangapark.org/title/114972-aria"),
("https://mangapark.io/title/114972-aria"),
("https://mangapark.me/title/114972-aria"),
)
def __init__(self, match): def __init__(self, match):
self.root = text.root_from_url(match.group(0)) self.root = text.root_from_url(match.group(0))

96
gallery_dl/extractor/mangaread.py
View File

@ -35,56 +35,7 @@ class MangareadChapterExtractor(MangareadBase, ChapterExtractor):
"""Extractor for manga-chapters from mangaread.org""" """Extractor for manga-chapters from mangaread.org"""
pattern = (r"(?:https?://)?(?:www\.)?mangaread\.org" pattern = (r"(?:https?://)?(?:www\.)?mangaread\.org"
r"(/manga/[^/?#]+/[^/?#]+)") r"(/manga/[^/?#]+/[^/?#]+)")
test = ( example = "https://www.mangaread.org/manga/MANGA/chapter-01/"
("https://www.mangaread.org/manga/one-piece/chapter-1053-3/", {
"pattern": (r"https://www\.mangaread\.org/wp-content/uploads"
r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"),
"count": 11,
"keyword": {
"manga" : "One Piece",
"title" : "",
"chapter" : 1053,
"chapter_minor": ".3",
"tags" : ["Oda Eiichiro"],
"lang" : "en",
"language": "English",
}
}),
("https://www.mangaread.org/manga/one-piece/chapter-1000000/", {
"exception": exception.NotFoundError,
}),
(("https://www.mangaread.org"
"/manga/kanan-sama-wa-akumade-choroi/chapter-10/"), {
"pattern": (r"https://www\.mangaread\.org/wp-content/uploads"
r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"),
"count": 9,
"keyword": {
"manga" : "Kanan-sama wa Akumade Choroi",
"title" : "",
"chapter" : 10,
"chapter_minor": "",
"tags" : list,
"lang" : "en",
"language": "English",
}
}),
# 'Chapter146.5'
# ^^ no whitespace
("https://www.mangaread.org/manga/above-all-gods/chapter146-5/", {
"pattern": (r"https://www\.mangaread\.org/wp-content/uploads"
r"/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+"),
"count": 6,
"keyword": {
"manga" : "Above All Gods",
"title" : "",
"chapter" : 146,
"chapter_minor": ".5",
"tags" : list,
"lang" : "en",
"language": "English",
}
}),
)
def metadata(self, page): def metadata(self, page):
tags = text.extr(page, 'class="wp-manga-tags-list">', '</div>') tags = text.extr(page, 'class="wp-manga-tags-list">', '</div>')
@ -108,50 +59,7 @@ class MangareadMangaExtractor(MangareadBase, MangaExtractor):
"""Extractor for manga from mangaread.org""" """Extractor for manga from mangaread.org"""
chapterclass = MangareadChapterExtractor chapterclass = MangareadChapterExtractor
pattern = r"(?:https?://)?(?:www\.)?mangaread\.org(/manga/[^/?#]+)/?$" pattern = r"(?:https?://)?(?:www\.)?mangaread\.org(/manga/[^/?#]+)/?$"
test = ( example = "https://www.mangaread.org/manga/MANGA"
("https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", {
"pattern": (r"https://www\.mangaread\.org/manga"
r"/kanan-sama-wa-akumade-choroi"
r"/chapter-\d+(-.+)?/"),
"count" : ">= 13",
"keyword": {
"manga" : "Kanan-sama wa Akumade Choroi",
"author" : ["nonco"],
"artist" : ["nonco"],
"type" : "Manga",
"genres" : ["Comedy", "Romance", "Shounen", "Supernatural"],
"rating" : float,
"release": 2022,
"status" : "OnGoing",
"lang" : "en",
"language" : "English",
"manga_alt" : list,
"description": str,
}
}),
("https://www.mangaread.org/manga/one-piece", {
"pattern": (r"https://www\.mangaread\.org/manga"
r"/one-piece/chapter-\d+(-.+)?/"),
"count" : ">= 1066",
"keyword": {
"manga" : "One Piece",
"author" : ["Oda Eiichiro"],
"artist" : ["Oda Eiichiro"],
"type" : "Manga",
"genres" : list,
"rating" : float,
"release": 1997,
"status" : "OnGoing",
"lang" : "en",
"language" : "English",
"manga_alt" : ["One Piece"],
"description": str,
}
}),
("https://www.mangaread.org/manga/doesnotexist", {
"exception": exception.HttpError,
}),
)
def chapters(self, page): def chapters(self, page):
if 'class="error404' in page: if 'class="error404' in page:

89
gallery_dl/extractor/mangasee.py
View File

@ -36,54 +36,7 @@ class MangaseeBase():
class MangaseeChapterExtractor(MangaseeBase, ChapterExtractor): class MangaseeChapterExtractor(MangaseeBase, ChapterExtractor):
pattern = (r"(?:https?://)?(mangasee123|manga4life)\.com" pattern = (r"(?:https?://)?(mangasee123|manga4life)\.com"
r"(/read-online/[^/?#]+\.html)") r"(/read-online/[^/?#]+\.html)")
test = ( example = "https://mangasee123.com/read-online/MANGA-chapter-1-page-1.html"
(("https://mangasee123.com/read-online"
"/Tokyo-Innocent-chapter-4.5-page-1.html"), {
"pattern": r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png",
"count": 8,
"keyword": {
"author": ["NARUMI Naru"],
"chapter": 4,
"chapter_minor": ".5",
"chapter_string": "100045",
"count": 8,
"date": "dt:2020-01-20 21:52:53",
"extension": "png",
"filename": r"re:0004\.5-00\d",
"genre": ["Comedy", "Fantasy", "Harem", "Romance", "Shounen",
"Supernatural"],
"index": "1",
"lang": "en",
"language": "English",
"manga": "Tokyo Innocent",
"page": int,
"title": "",
},
}),
(("https://manga4life.com/read-online"
"/One-Piece-chapter-1063-page-1.html"), {
"pattern": r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png",
"count": 13,
"keyword": {
"author": ["ODA Eiichiro"],
"chapter": 1063,
"chapter_minor": "",
"chapter_string": "110630",
"count": 13,
"date": "dt:2022-10-16 17:32:54",
"extension": "png",
"filename": r"re:1063-0\d\d",
"genre": ["Action", "Adventure", "Comedy", "Drama", "Fantasy",
"Shounen"],
"index": "1",
"lang": "en",
"language": "English",
"manga": "One Piece",
"page": int,
"title": "",
},
}),
)
def __init__(self, match): def __init__(self, match):
if match.group(1) == "manga4life": if match.group(1) == "manga4life":
@ -134,45 +87,7 @@ class MangaseeChapterExtractor(MangaseeBase, ChapterExtractor):
class MangaseeMangaExtractor(MangaseeBase, MangaExtractor): class MangaseeMangaExtractor(MangaseeBase, MangaExtractor):
chapterclass = MangaseeChapterExtractor chapterclass = MangaseeChapterExtractor
pattern = r"(?:https?://)?(mangasee123|manga4life)\.com(/manga/[^/?#]+)" pattern = r"(?:https?://)?(mangasee123|manga4life)\.com(/manga/[^/?#]+)"
test = ( example = "https://mangasee123.com/manga/MANGA"
(("https://mangasee123.com/manga"
"/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai"), {
"pattern": MangaseeChapterExtractor.pattern,
"count": ">= 17",
"keyword": {
"author": ["TAKASE Masaya"],
"chapter": int,
"chapter_minor": r"re:^|\.5$",
"chapter_string": r"re:100\d\d\d",
"date": "type:datetime",
"genre": ["Comedy", "Romance", "School Life", "Shounen",
"Slice of Life"],
"index": "1",
"lang": "en",
"language": "English",
"manga": "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai",
"title": "",
},
}),
("https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", {
"pattern": MangaseeChapterExtractor.pattern,
"count": ">= 50",
"keyword": {
"author": ["Canno"],
"chapter": int,
"chapter_minor": r"re:^|\.5$",
"chapter_string": r"re:100\d\d\d",
"date": "type:datetime",
"genre": ["Comedy", "Romance", "School Life", "Seinen",
"Shoujo Ai"],
"index": "1",
"lang": "en",
"language": "English",
"manga": "Ano-Musume-Ni-Kiss-To-Shirayuri-O",
"title": ""
},
}),
)
def __init__(self, match): def __init__(self, match):
if match.group(1) == "manga4life": if match.group(1) == "manga4life":

28
gallery_dl/extractor/mangoxo.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2019-2020 Mike Fährmann # Copyright 2019-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -80,25 +80,7 @@ class MangoxoAlbumExtractor(MangoxoExtractor):
directory_fmt = ("{category}", "{channel[name]}", "{album[name]}") directory_fmt = ("{category}", "{channel[name]}", "{album[name]}")
archive_fmt = "{album[id]}_{num}" archive_fmt = "{album[id]}_{num}"
pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/album/(\w+)" pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/album/(\w+)"
test = ("https://www.mangoxo.com/album/lzVOv1Q9", { example = "https://www.mangoxo.com/album/ID"
"url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d",
"keyword": {
"channel": {
"id": "gaxO16d8",
"name": "Phoenix",
"cover": str,
},
"album": {
"id": "lzVOv1Q9",
"name": "re:池永康晟 Ikenaga Yasunari 透出古朴",
"date": "dt:2019-03-22 14:42:00",
"description": str,
},
"id": int,
"num": int,
"count": 65,
},
})
def __init__(self, match): def __init__(self, match):
MangoxoExtractor.__init__(self, match) MangoxoExtractor.__init__(self, match)
@ -163,11 +145,7 @@ class MangoxoChannelExtractor(MangoxoExtractor):
"""Extractor for all albums on a mangoxo channel""" """Extractor for all albums on a mangoxo channel"""
subcategory = "channel" subcategory = "channel"
pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/(\w+)/album" pattern = r"(?:https?://)?(?:www\.)?mangoxo\.com/(\w+)/album"
test = ("https://www.mangoxo.com/phoenix/album", { example = "https://www.mangoxo.com/USER/album"
"pattern": MangoxoAlbumExtractor.pattern,
"range": "1-30",
"count": "> 20",
})
def __init__(self, match): def __init__(self, match):
MangoxoExtractor.__init__(self, match) MangoxoExtractor.__init__(self, match)

56
gallery_dl/extractor/mastodon.py
View File

@ -106,29 +106,7 @@ class MastodonUserExtractor(MastodonExtractor):
"""Extractor for all images of an account/user""" """Extractor for all images of an account/user"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/(?:@|users/)([^/?#]+)(?:/media)?/?$" pattern = BASE_PATTERN + r"/(?:@|users/)([^/?#]+)(?:/media)?/?$"
test = ( example = "https://mastodon.social/@USER"
("https://mastodon.social/@jk", {
"pattern": r"https://files.mastodon.social/media_attachments"
r"/files/(\d+/){3,}original/\w+",
"range": "1-60",
"count": 60,
}),
("https://pawoo.net/@yoru_nine/", {
"range": "1-60",
"count": 60,
}),
("https://baraag.net/@pumpkinnsfw"),
("https://mastodon.social/@yoru_nine@pawoo.net", {
"pattern": r"https://mastodon\.social/media_proxy/\d+/original",
"range": "1-10",
"count": 10,
}),
("https://mastodon.social/@id:10843"),
("https://mastodon.social/users/id:10843"),
("https://mastodon.social/users/jk"),
("https://mastodon.social/users/yoru_nine@pawoo.net"),
("https://mastodon.social/web/@jk"),
)
def statuses(self): def statuses(self):
api = MastodonAPI(self) api = MastodonAPI(self)
@ -144,11 +122,7 @@ class MastodonBookmarkExtractor(MastodonExtractor):
"""Extractor for mastodon bookmarks""" """Extractor for mastodon bookmarks"""
subcategory = "bookmark" subcategory = "bookmark"
pattern = BASE_PATTERN + r"/bookmarks" pattern = BASE_PATTERN + r"/bookmarks"
test = ( example = "https://mastodon.social/bookmarks"
("https://mastodon.social/bookmarks"),
("https://pawoo.net/bookmarks"),
("https://baraag.net/bookmarks"),
)
def statuses(self): def statuses(self):
return MastodonAPI(self).account_bookmarks() return MastodonAPI(self).account_bookmarks()
@ -158,15 +132,7 @@ class MastodonFollowingExtractor(MastodonExtractor):
"""Extractor for followed mastodon users""" """Extractor for followed mastodon users"""
subcategory = "following" subcategory = "following"
pattern = BASE_PATTERN + r"/users/([^/?#]+)/following" pattern = BASE_PATTERN + r"/users/([^/?#]+)/following"
test = ( example = "https://mastodon.social/users/USER/following"
("https://mastodon.social/users/0x4f/following", {
"extractor": False,
"count": ">= 20",
}),
("https://mastodon.social/users/id:10843/following"),
("https://pawoo.net/users/yoru_nine/following"),
("https://baraag.net/users/pumpkinnsfw/following"),
)
def items(self): def items(self):
api = MastodonAPI(self) api = MastodonAPI(self)
@ -181,21 +147,7 @@ class MastodonStatusExtractor(MastodonExtractor):
"""Extractor for images from a status""" """Extractor for images from a status"""
subcategory = "status" subcategory = "status"
pattern = BASE_PATTERN + r"/@[^/?#]+/(\d+)" pattern = BASE_PATTERN + r"/@[^/?#]+/(\d+)"
test = ( example = "https://mastodon.social/@USER/12345"
("https://mastodon.social/@jk/103794036899778366", {
"count": 4,
"keyword": {
"count": 4,
"num": int,
},
}),
("https://pawoo.net/@yoru_nine/105038878897832922", {
"content": "b52e807f8ab548d6f896b09218ece01eba83987a",
}),
("https://baraag.net/@pumpkinnsfw/104364170556898443", {
"content": "67748c1b828c58ad60d0fe5729b59fb29c872244",
}),
)
def statuses(self): def statuses(self):
return (MastodonAPI(self).status(self.item),) return (MastodonAPI(self).status(self.item),)

50
gallery_dl/extractor/misskey.py
View File

@ -85,24 +85,7 @@ class MisskeyUserExtractor(MisskeyExtractor):
"""Extractor for all images of a Misskey user""" """Extractor for all images of a Misskey user"""
subcategory = "user" subcategory = "user"
pattern = BASE_PATTERN + r"/@([^/?#]+)/?$" pattern = BASE_PATTERN + r"/@([^/?#]+)/?$"
test = ( example = "https://misskey.io/@USER"
("https://misskey.io/@lithla", {
"pattern": r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+",
"range": "1-50",
"count": 50,
}),
("https://misskey.io/@blooddj@pawoo.net", {
"range": "1-50",
"count": 50,
}),
("https://lesbian.energy/@rerorero", {
"pattern": r"https://lesbian.energy/files/\w+",
"range": "1-50",
"count": 50,
}),
("https://lesbian.energy/@nano@mk.yopo.work"),
("https://sushi.ski/@ui@misskey.04.si"),
)
def notes(self): def notes(self):
return self.api.users_notes(self.api.user_id_by_username(self.item)) return self.api.users_notes(self.api.user_id_by_username(self.item))
@ -112,13 +95,7 @@ class MisskeyFollowingExtractor(MisskeyExtractor):
"""Extractor for followed Misskey users""" """Extractor for followed Misskey users"""
subcategory = "following" subcategory = "following"
pattern = BASE_PATTERN + r"/@([^/?#]+)/following" pattern = BASE_PATTERN + r"/@([^/?#]+)/following"
test = ( example = "https://misskey.io/@USER/following"
("https://misskey.io/@blooddj@pawoo.net/following", {
"extractor": False,
"count": ">= 6",
}),
("https://sushi.ski/@hatusimo_sigure/following"),
)
def items(self): def items(self):
user_id = self.api.user_id_by_username(self.item) user_id = self.api.user_id_by_username(self.item)
@ -136,21 +113,7 @@ class MisskeyNoteExtractor(MisskeyExtractor):
"""Extractor for images from a Note""" """Extractor for images from a Note"""
subcategory = "note" subcategory = "note"
pattern = BASE_PATTERN + r"/notes/(\w+)" pattern = BASE_PATTERN + r"/notes/(\w+)"
test = ( example = "https://misskey.io/notes/98765"
("https://misskey.io/notes/9bhqfo835v", {
"pattern": r"https://s\d+\.arkjp\.net/misskey/[\w-]+\.\w+",
"count": 4,
}),
("https://misskey.io/notes/9brq7z1re6"),
("https://sushi.ski/notes/9bm3x4ksqw", {
"pattern": r"https://media\.sushi\.ski/files/[\w-]+\.png",
"count": 1,
}),
("https://lesbian.energy/notes/995ig09wqy", {
"count": 1,
}),
("https://lesbian.energy/notes/96ynd9w5kc"),
)
def notes(self): def notes(self):
return (self.api.notes_show(self.item),) return (self.api.notes_show(self.item),)
@ -160,12 +123,7 @@ class MisskeyFavoriteExtractor(MisskeyExtractor):
"""Extractor for favorited notes""" """Extractor for favorited notes"""
subcategory = "favorite" subcategory = "favorite"
pattern = BASE_PATTERN + r"/(?:my|api/i)/favorites" pattern = BASE_PATTERN + r"/(?:my|api/i)/favorites"
test = ( example = "https://misskey.io/my/favorites"
("https://misskey.io/my/favorites"),
("https://misskey.io/api/i/favorites"),
("https://lesbian.energy/my/favorites"),
("https://sushi.ski/my/favorites"),
)
def notes(self): def notes(self):
return self.api.i_favorites() return self.api.i_favorites()

125
gallery_dl/extractor/moebooru.py
View File

@ -92,94 +92,12 @@ BASE_PATTERN = MoebooruExtractor.update({
}) })
class MoebooruPostExtractor(MoebooruExtractor):
subcategory = "post"
archive_fmt = "{id}"
pattern = BASE_PATTERN + r"/post/show/(\d+)"
test = (
("https://yande.re/post/show/51824", {
"content": "59201811c728096b2d95ce6896fd0009235fe683",
"options": (("tags", True),),
"keyword": {
"tags_artist": "sasaki_tamaru",
"tags_circle": "softhouse_chara",
"tags_copyright": "ouzoku",
"tags_general": str,
},
}),
("https://konachan.com/post/show/205189", {
"content": "674e75a753df82f5ad80803f575818b8e46e4b65",
"options": (("tags", True),),
"keyword": {
"tags_artist": "patata",
"tags_character": "clownpiece",
"tags_copyright": "touhou",
"tags_general": str,
},
}),
("https://yande.re/post/show/993156", {
"content": "fed722bd90f48de41ec163692befc701056e2b1e",
"options": (("notes", True),),
"keyword": {
"notes": [
{
"id": 7096,
"x" : 90,
"y" : 626,
"width" : 283,
"height": 529,
"body" : "Please keep this as a secret for me!!",
},
{
"id": 7095,
"x" : 900,
"y" : 438,
"width" : 314,
"height": 588,
"body" : "The facts that I love playing games",
},
],
},
}),
("https://lolibooru.moe/post/show/281305/", {
"content": "a331430223ffc5b23c31649102e7d49f52489b57",
"options": (("notes", True),),
"keyword": {
"notes": list,
},
}),
("https://konachan.net/post/show/205189"),
("https://www.sakugabooru.com/post/show/125570"),
("https://lolibooru.moe/post/show/287835"),
)
def __init__(self, match):
MoebooruExtractor.__init__(self, match)
self.post_id = match.group(match.lastindex)
def posts(self):
params = {"tags": "id:" + self.post_id}
return self.request(self.root + "/post.json", params=params).json()
class MoebooruTagExtractor(MoebooruExtractor): class MoebooruTagExtractor(MoebooruExtractor):
subcategory = "tag" subcategory = "tag"
directory_fmt = ("{category}", "{search_tags}") directory_fmt = ("{category}", "{search_tags}")
archive_fmt = "t_{search_tags}_{id}" archive_fmt = "t_{search_tags}_{id}"
pattern = BASE_PATTERN + r"/post\?(?:[^&#]*&)*tags=([^&#]*)" pattern = BASE_PATTERN + r"/post\?(?:[^&#]*&)*tags=([^&#]*)"
test = ( example = "https://yande.re/post?tags=TAG"
("https://yande.re/post?tags=ouzoku+armor", {
"content": "59201811c728096b2d95ce6896fd0009235fe683",
}),
("https://konachan.com/post?tags=patata", {
"content": "838cfb815e31f48160855435655ddf7bfc4ecb8d",
}),
# empty 'tags' (#4354)
("https://konachan.com/post?tags="),
("https://konachan.net/post?tags=patata"),
("https://www.sakugabooru.com/post?tags=nichijou"),
("https://lolibooru.moe/post?tags=ruu_%28tksymkw%29"),
)
def __init__(self, match): def __init__(self, match):
MoebooruExtractor.__init__(self, match) MoebooruExtractor.__init__(self, match)
@ -199,17 +117,7 @@ class MoebooruPoolExtractor(MoebooruExtractor):
directory_fmt = ("{category}", "pool", "{pool}") directory_fmt = ("{category}", "pool", "{pool}")
archive_fmt = "p_{pool}_{id}" archive_fmt = "p_{pool}_{id}"
pattern = BASE_PATTERN + r"/pool/show/(\d+)" pattern = BASE_PATTERN + r"/pool/show/(\d+)"
test = ( example = "https://yande.re/pool/show/12345"
("https://yande.re/pool/show/318", {
"content": "2a35b9d6edecce11cc2918c6dce4de2198342b68",
}),
("https://konachan.com/pool/show/95", {
"content": "cf0546e38a93c2c510a478f8744e60687b7a8426",
}),
("https://konachan.net/pool/show/95"),
("https://www.sakugabooru.com/pool/show/54"),
("https://lolibooru.moe/pool/show/239"),
)
def __init__(self, match): def __init__(self, match):
MoebooruExtractor.__init__(self, match) MoebooruExtractor.__init__(self, match)
@ -223,25 +131,28 @@ class MoebooruPoolExtractor(MoebooruExtractor):
return self._pagination(self.root + "/post.json", params) return self._pagination(self.root + "/post.json", params)
class MoebooruPostExtractor(MoebooruExtractor):
subcategory = "post"
archive_fmt = "{id}"
pattern = BASE_PATTERN + r"/post/show/(\d+)"
example = "https://yande.re/post/show/12345"
def __init__(self, match):
MoebooruExtractor.__init__(self, match)
self.post_id = match.group(match.lastindex)
def posts(self):
params = {"tags": "id:" + self.post_id}
return self.request(self.root + "/post.json", params=params).json()
class MoebooruPopularExtractor(MoebooruExtractor): class MoebooruPopularExtractor(MoebooruExtractor):
subcategory = "popular" subcategory = "popular"
directory_fmt = ("{category}", "popular", "{scale}", "{date}") directory_fmt = ("{category}", "popular", "{scale}", "{date}")
archive_fmt = "P_{scale[0]}_{date}_{id}" archive_fmt = "P_{scale[0]}_{date}_{id}"
pattern = BASE_PATTERN + \ pattern = BASE_PATTERN + \
r"/post/popular_(by_(?:day|week|month)|recent)(?:\?([^#]*))?" r"/post/popular_(by_(?:day|week|month)|recent)(?:\?([^#]*))?"
test = ( example = "https://yande.re/post/popular_by_month?year=YYYY&month=MM"
("https://yande.re/post/popular_by_month?month=6&year=2014", {
"count": 40,
}),
("https://yande.re/post/popular_recent"),
("https://konachan.com/post/popular_by_month?month=11&year=2010", {
"count": 20,
}),
("https://konachan.com/post/popular_recent"),
("https://konachan.net/post/popular_recent"),
("https://www.sakugabooru.com/post/popular_recent"),
("https://lolibooru.moe/post/popular_recent"),
)
def __init__(self, match): def __init__(self, match):
MoebooruExtractor.__init__(self, match) MoebooruExtractor.__init__(self, match)

19
gallery_dl/extractor/myhentaigallery.py
View File

@ -13,26 +13,11 @@ from .. import text, exception
class MyhentaigalleryGalleryExtractor(GalleryExtractor): class MyhentaigalleryGalleryExtractor(GalleryExtractor):
"""Extractor for image galleries from myhentaigallery.com""" """Extractor for image galleries from myhentaigallery.com"""
category = "myhentaigallery" category = "myhentaigallery"
root = "https://myhentaigallery.com"
directory_fmt = ("{category}", "{gallery_id} {artist:?[/] /J, }{title}") directory_fmt = ("{category}", "{gallery_id} {artist:?[/] /J, }{title}")
pattern = (r"(?:https?://)?myhentaigallery\.com" pattern = (r"(?:https?://)?myhentaigallery\.com"
r"/gallery/(?:thumbnails|show)/(\d+)") r"/gallery/(?:thumbnails|show)/(\d+)")
test = ( example = "https://myhentaigallery.com/gallery/thumbnails/12345"
("https://myhentaigallery.com/gallery/thumbnails/16247", {
"pattern": r"https://images.myhentaicomics\.com/imagesgallery"
r"/images/[^/]+/original/\d+\.jpg",
"keyword": {
"artist" : list,
"count" : 11,
"gallery_id": 16247,
"group" : list,
"parodies" : list,
"tags" : ["Giantess"],
"title" : "Attack Of The 50ft Woman 1",
},
}),
("https://myhentaigallery.com/gallery/show/16247/1"),
)
root = "https://myhentaigallery.com"
def __init__(self, match): def __init__(self, match):
self.gallery_id = match.group(1) self.gallery_id = match.group(1)

22
gallery_dl/extractor/myportfolio.py
View File

@ -22,27 +22,7 @@ class MyportfolioGalleryExtractor(Extractor):
pattern = (r"(?:myportfolio:(?:https?://)?([^/]+)|" pattern = (r"(?:myportfolio:(?:https?://)?([^/]+)|"
r"(?:https?://)?([\w-]+\.myportfolio\.com))" r"(?:https?://)?([\w-]+\.myportfolio\.com))"
r"(/[^/?#]+)?") r"(/[^/?#]+)?")
test = ( example = "https://USER.myportfolio.com/TITLE"
("https://andrewling.myportfolio.com/volvo-xc-90-hybrid", {
"url": "acea0690c76db0e5cf267648cefd86e921bc3499",
"keyword": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d",
}),
("https://andrewling.myportfolio.com/", {
"pattern": r"https://andrewling\.myportfolio\.com/[^/?#+]+$",
"count": ">= 6",
}),
("https://stevenilousphotography.myportfolio.com/society", {
"exception": exception.NotFoundError,
}),
# custom domain
("myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", {
"count": 3,
}),
("myportfolio:https://tooco.com.ar/", {
"pattern": pattern,
"count": ">= 40",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

27
gallery_dl/extractor/naver.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2019 Mike Fährmann # Copyright 2019-2023 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@ -27,17 +27,7 @@ class NaverPostExtractor(NaverBase, GalleryExtractor):
archive_fmt = "{blog[id]}_{post[num]}_{num}" archive_fmt = "{blog[id]}_{post[num]}_{num}"
pattern = (r"(?:https?://)?blog\.naver\.com/" pattern = (r"(?:https?://)?blog\.naver\.com/"
r"(?:PostView\.nhn\?blogId=(\w+)&logNo=(\d+)|(\w+)/(\d+)/?$)") r"(?:PostView\.nhn\?blogId=(\w+)&logNo=(\d+)|(\w+)/(\d+)/?$)")
test = ( example = "https://blog.naver.com/BLOGID/12345"
("https://blog.naver.com/rlfqjxm0/221430673006", {
"url": "6c694f3aced075ed5e9511f1e796d14cb26619cc",
"keyword": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e",
}),
(("https://blog.naver.com/PostView.nhn"
"?blogId=rlfqjxm0&logNo=221430673006"), {
"url": "6c694f3aced075ed5e9511f1e796d14cb26619cc",
"keyword": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e",
}),
)
def __init__(self, match): def __init__(self, match):
blog_id = match.group(1) blog_id = match.group(1)
@ -84,18 +74,7 @@ class NaverBlogExtractor(NaverBase, Extractor):
categorytransfer = True categorytransfer = True
pattern = (r"(?:https?://)?blog\.naver\.com/" pattern = (r"(?:https?://)?blog\.naver\.com/"
r"(?:PostList.nhn\?(?:[^&#]+&)*blogId=([^&#]+)|(\w+)/?$)") r"(?:PostList.nhn\?(?:[^&#]+&)*blogId=([^&#]+)|(\w+)/?$)")
test = ( example = "https://blog.naver.com/BLOGID"
("https://blog.naver.com/gukjung", {
"pattern": NaverPostExtractor.pattern,
"count": 12,
"range": "1-12",
}),
("https://blog.naver.com/PostList.nhn?blogId=gukjung", {
"pattern": NaverPostExtractor.pattern,
"count": 12,
"range": "1-12",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

64
gallery_dl/extractor/naverwebtoon.py
View File

@ -28,54 +28,7 @@ class NaverwebtoonEpisodeExtractor(NaverwebtoonBase, GalleryExtractor):
filename_fmt = "{episode:>03}-{num:>02}.{extension}" filename_fmt = "{episode:>03}-{num:>02}.{extension}"
archive_fmt = "{title_id}_{episode}_{num}" archive_fmt = "{title_id}_{episode}_{num}"
pattern = BASE_PATTERN + r"/detail(?:\.nhn)?\?([^#]+)" pattern = BASE_PATTERN + r"/detail(?:\.nhn)?\?([^#]+)"
test = ( example = "https://comic.naver.com/webtoon/detail?titleId=12345&no=1"
(("https://comic.naver.com/webtoon/detail"
"?titleId=26458&no=1&weekday=tue"), {
"url": "47a956ba8c7a837213d5985f50c569fcff986f75",
"content": "3806b6e8befbb1920048de9888dfce6220f69a60",
"count": 14,
"keyword": {
"author": ["김규삼"],
"artist": ["김규삼"],
"comic": "N의등대-눈의등대",
"count": 14,
"episode": "1",
"extension": "jpg",
"num": int,
"tags": ["스릴러", "완결무료", "완결스릴러"],
"title": "n의 등대 - 눈의 등대 1화",
"title_id": "26458",
},
}),
(("https://comic.naver.com/challenge/detail"
"?titleId=765124&no=1"), {
"pattern": r"https://image-comic\.pstatic\.net"
r"/user_contents_data/challenge_comic/2021/01/19"
r"/342586/upload_7149856273586337846\.jpeg",
"count": 1,
"keyword": {
"author": ["kemi****"],
"artist": [],
"comic": "우니 모두의 이야기",
"count": 1,
"episode": "1",
"extension": "jpeg",
"filename": "upload_7149856273586337846",
"num": 1,
"tags": ["일상툰", "우니모두의이야기", "퇴사", "입사", "신입사원",
"사회초년생", "회사원", "20대"],
"title": "퇴사하다",
"title_id": "765124",
},
}),
(("https://comic.naver.com/bestChallenge/detail.nhn"
"?titleId=771467&no=3"), {
"pattern": r"https://image-comic\.pstatic\.net"
r"/user_contents_data/challenge_comic/2021/04/28"
r"/345534/upload_3617293622396203109\.jpeg",
"count": 1,
}),
)
def __init__(self, match): def __init__(self, match):
path, query = match.groups() path, query = match.groups()
@ -115,20 +68,7 @@ class NaverwebtoonComicExtractor(NaverwebtoonBase, Extractor):
subcategory = "comic" subcategory = "comic"
categorytransfer = True categorytransfer = True
pattern = BASE_PATTERN + r"/list(?:\.nhn)?\?([^#]+)" pattern = BASE_PATTERN + r"/list(?:\.nhn)?\?([^#]+)"
test = ( example = "https://comic.naver.com/webtoon/list?titleId=12345"
("https://comic.naver.com/webtoon/list?titleId=22073", {
"pattern": NaverwebtoonEpisodeExtractor.pattern,
"count": 32,
}),
("https://comic.naver.com/challenge/list?titleId=765124", {
"pattern": NaverwebtoonEpisodeExtractor.pattern,
"count": 25,
}),
("https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", {
"pattern": NaverwebtoonEpisodeExtractor.pattern,
"count": ">= 12",
}),
)
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)

200
gallery_dl/extractor/newgrounds.py
View File

@ -300,41 +300,7 @@ class NewgroundsImageExtractor(NewgroundsExtractor):
pattern = (r"(?:https?://)?(?:" pattern = (r"(?:https?://)?(?:"
r"(?:www\.)?newgrounds\.com/art/view/([^/?#]+)/[^/?#]+" r"(?:www\.)?newgrounds\.com/art/view/([^/?#]+)/[^/?#]+"
r"|art\.ngfiles\.com/images/\d+/\d+_([^_]+)_([^.]+))") r"|art\.ngfiles\.com/images/\d+/\d+_([^_]+)_([^.]+))")
test = ( example = "https://www.newgrounds.com/art/view/USER/TITLE"
("https://www.newgrounds.com/art/view/tomfulp/ryu-is-hawt", {
"url": "57f182bcbbf2612690c3a54f16ffa1da5105245e",
"content": "8f395e08333eb2457ba8d8b715238f8910221365",
"keyword": {
"artist" : ["tomfulp"],
"comment" : "re:Consider this the bottom threshold for ",
"date" : "dt:2009-06-04 14:44:05",
"description": "re:Consider this the bottom threshold for ",
"favorites" : int,
"filename" : "94_tomfulp_ryu-is-hawt",
"height" : 476,
"index" : 94,
"rating" : "e",
"score" : float,
"tags" : ["ryu", "streetfighter"],
"title" : "Ryu is Hawt",
"type" : "article",
"user" : "tomfulp",
"width" : 447,
},
}),
("https://art.ngfiles.com/images/0/94_tomfulp_ryu-is-hawt.gif", {
"url": "57f182bcbbf2612690c3a54f16ffa1da5105245e",
}),
("https://www.newgrounds.com/art/view/sailoryon/yon-dream-buster", {
"url": "84eec95e663041a80630df72719f231e157e5f5d",
"count": 2,
}),
# "adult" rated (#2456)
("https://www.newgrounds.com/art/view/kekiiro/red", {
"options": (("username", None),),
"count": 1,
}),
)
def __init__(self, match): def __init__(self, match):
NewgroundsExtractor.__init__(self, match) NewgroundsExtractor.__init__(self, match)
@ -354,104 +320,7 @@ class NewgroundsMediaExtractor(NewgroundsExtractor):
subcategory = "media" subcategory = "media"
pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com" pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com"
r"(/(?:portal/view|audio/listen)/\d+)") r"(/(?:portal/view|audio/listen)/\d+)")
test = ( example = "https://www.newgrounds.com/portal/view/12345"
("https://www.newgrounds.com/portal/view/595355", {
"pattern": r"https://uploads\.ungrounded\.net/alternate/564000"
r"/564957_alternate_31\.mp4\?1359712249",
"keyword": {
"artist" : ["kickinthehead", "danpaladin", "tomfulp"],
"comment" : "re:My fan trailer for Alien Hominid HD!",
"date" : "dt:2013-02-01 09:50:49",
"description": "Fan trailer for Alien Hominid HD!",
"favorites" : int,
"filename" : "564957_alternate_31",
"index" : 595355,
"rating" : "e",
"score" : float,
"tags" : ["alienhominid", "trailer"],
"title" : "Alien Hominid Fan Trailer",
"type" : "movie",
"user" : "kickinthehead",
},
}),
("https://www.newgrounds.com/audio/listen/609768", {
"url": "f4c5490ae559a3b05e46821bb7ee834f93a43c95",
"keyword": {
"artist" : ["zj", "tomfulp"],
"comment" : "re:RECORDED 12-09-2014\n\nFrom The ZJ \"Late ",
"date" : "dt:2015-02-23 19:31:59",
"description": "From The ZJ Report Show!",
"favorites" : int,
"index" : 609768,
"rating" : "",
"score" : float,
"tags" : ["fulp", "interview", "tom", "zj"],
"title" : "ZJ Interviews Tom Fulp!",
"type" : "music.song",
"user" : "zj",
},
}),
# flash animation (#1257)
("https://www.newgrounds.com/portal/view/161181/format/flash", {
"pattern": r"https://uploads\.ungrounded\.net/161000"
r"/161181_ddautta_mask__550x281_\.swf\?f1081628129",
"keyword": {"type": "movie"},
}),
# format selection (#1729)
("https://www.newgrounds.com/portal/view/758545", {
"options": (("format", "720p"),),
"pattern": r"https://uploads\.ungrounded\.net/alternate/1482000"
r"/1482860_alternate_102516\.720p\.mp4\?\d+",
}),
# "adult" rated (#2456)
("https://www.newgrounds.com/portal/view/717744", {
"options": (("username", None),),
"count": 1,
}),
# flash game
("https://www.newgrounds.com/portal/view/829032", {
"pattern": r"https://uploads\.ungrounded\.net/829000"
r"/829032_picovsbeardx\.swf\?f1641968445",
"range": "1",
"keyword": {
"artist" : [
"dungeonation",
"carpetbakery",
"animalspeakandrews",
"bill",
"chipollo",
"dylz49",
"gappyshamp",
"pinktophat",
"rad",
"shapeshiftingblob",
"tomfulp",
"voicesbycorey",
"psychogoldfish",
],
"comment" : "re:The children are expendable. Take out the ",
"date" : "dt:2022-01-10 23:00:57",
"description": "Bloodshed in The Big House that Blew...again!",
"favorites" : int,
"index" : 829032,
"post_url" : "https://www.newgrounds.com/portal/view/829032",
"rating" : "m",
"score" : float,
"tags" : [
"assassin",
"boyfriend",
"darnell",
"nene",
"pico",
"picos-school",
],
"title" : "PICO VS BEAR DX",
"type" : "game",
"url" : "https://uploads.ungrounded.net/829000"
"/829032_picovsbeardx.swf?f1641968445",
},
}),
)
def __init__(self, match): def __init__(self, match):
NewgroundsExtractor.__init__(self, match) NewgroundsExtractor.__init__(self, match)
@ -466,58 +335,35 @@ class NewgroundsArtExtractor(NewgroundsExtractor):
"""Extractor for all images of a newgrounds user""" """Extractor for all images of a newgrounds user"""
subcategory = _path = "art" subcategory = _path = "art"
pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/art/?$" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/art/?$"
test = ("https://tomfulp.newgrounds.com/art", { example = "https://USER.newgrounds.com/art"
"pattern": NewgroundsImageExtractor.pattern,
"count": ">= 3",
})
class NewgroundsAudioExtractor(NewgroundsExtractor): class NewgroundsAudioExtractor(NewgroundsExtractor):
"""Extractor for all audio submissions of a newgrounds user""" """Extractor for all audio submissions of a newgrounds user"""
subcategory = _path = "audio" subcategory = _path = "audio"
pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/audio/?$" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/audio/?$"
test = ("https://tomfulp.newgrounds.com/audio", { example = "https://USER.newgrounds.com/audio"
"pattern": r"https://audio.ngfiles.com/\d+/\d+_.+\.mp3",
"count": ">= 4",
})
class NewgroundsMoviesExtractor(NewgroundsExtractor): class NewgroundsMoviesExtractor(NewgroundsExtractor):
"""Extractor for all movies of a newgrounds user""" """Extractor for all movies of a newgrounds user"""
subcategory = _path = "movies" subcategory = _path = "movies"
pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/movies/?$" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/movies/?$"
test = ("https://tomfulp.newgrounds.com/movies", { example = "https://USER.newgrounds.com/movies<"
"pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+",
"range": "1-10",
"count": 10,
})
class NewgroundsGamesExtractor(NewgroundsExtractor): class NewgroundsGamesExtractor(NewgroundsExtractor):
"""Extractor for a newgrounds user's games""" """Extractor for a newgrounds user's games"""
subcategory = _path = "games" subcategory = _path = "games"
pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/games/?$" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/games/?$"
test = ("https://tomfulp.newgrounds.com/games", { example = "https://USER.newgrounds.com/games"
"pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+",
"range": "1-10",
"count": 10,
})
class NewgroundsUserExtractor(NewgroundsExtractor): class NewgroundsUserExtractor(NewgroundsExtractor):
"""Extractor for a newgrounds user profile""" """Extractor for a newgrounds user profile"""
subcategory = "user" subcategory = "user"
pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/?$" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/?$"
test = ( example = "https://USER.newgrounds.com"
("https://tomfulp.newgrounds.com", {
"pattern": "https://tomfulp.newgrounds.com/art$",
}),
("https://tomfulp.newgrounds.com", {
"options": (("include", "all"),),
"pattern": "https://tomfulp.newgrounds.com/(art|audio|movies)$",
"count": 3,
}),
)
def initialize(self): def initialize(self):
pass pass
@ -538,15 +384,7 @@ class NewgroundsFavoriteExtractor(NewgroundsExtractor):
directory_fmt = ("{category}", "{user}", "Favorites") directory_fmt = ("{category}", "{user}", "Favorites")
pattern = (r"(?:https?://)?([\w-]+)\.newgrounds\.com" pattern = (r"(?:https?://)?([\w-]+)\.newgrounds\.com"
r"/favorites(?!/following)(?:/(art|audio|movies))?/?") r"/favorites(?!/following)(?:/(art|audio|movies))?/?")
test = ( example = "https://USER.newgrounds.com/favorites"
("https://tomfulp.newgrounds.com/favorites/art", {
"range": "1-10",
"count": ">= 10",
}),
("https://tomfulp.newgrounds.com/favorites/audio"),
("https://tomfulp.newgrounds.com/favorites/movies"),
("https://tomfulp.newgrounds.com/favorites/"),
)
def __init__(self, match): def __init__(self, match):
NewgroundsExtractor.__init__(self, match) NewgroundsExtractor.__init__(self, match)
@ -595,11 +433,7 @@ class NewgroundsFollowingExtractor(NewgroundsFavoriteExtractor):
"""Extractor for a newgrounds user's favorited users""" """Extractor for a newgrounds user's favorited users"""
subcategory = "following" subcategory = "following"
pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/favorites/(following)" pattern = r"(?:https?://)?([\w-]+)\.newgrounds\.com/favorites/(following)"
test = ("https://tomfulp.newgrounds.com/favorites/following", { example = "https://USER.newgrounds.com/favorites/following"
"pattern": NewgroundsUserExtractor.pattern,
"range": "76-125",
"count": 50,
})
def items(self): def items(self):
data = {"_extractor": NewgroundsUserExtractor} data = {"_extractor": NewgroundsUserExtractor}
@ -620,21 +454,7 @@ class NewgroundsSearchExtractor(NewgroundsExtractor):
directory_fmt = ("{category}", "search", "{search_tags}") directory_fmt = ("{category}", "search", "{search_tags}")
pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com" pattern = (r"(?:https?://)?(?:www\.)?newgrounds\.com"
r"/search/conduct/([^/?#]+)/?\?([^#]+)") r"/search/conduct/([^/?#]+)/?\?([^#]+)")
test = ( example = "https://www.newgrounds.com/search/conduct/art?terms=QUERY"
("https://www.newgrounds.com/search/conduct/art?terms=tree", {
"pattern": NewgroundsImageExtractor.pattern,
"keyword": {"search_tags": "tree"},
"range": "1-10",
"count": 10,
}),
("https://www.newgrounds.com/search/conduct/movies?terms=tree", {
"pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+",
"range": "1-10",
"count": 10,
}),
("https://www.newgrounds.com/search/conduct/audio?advanced=1"
"&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm"),
)
def __init__(self, match): def __init__(self, match):
NewgroundsExtractor.__init__(self, match) NewgroundsExtractor.__init__(self, match)

Some files were not shown because too many files have changed in this diff Show More