1
0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-22 10:42:34 +01:00

categorize sites in supportedsites.md by basecategory

This commit is contained in:
Mike Fährmann 2021-03-12 03:56:54 +01:00
parent a67e002f40
commit ec98b2c56f
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 241 additions and 189 deletions

View File

@ -31,12 +31,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Boards, Threads</td> <td>Boards, Threads</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>4plebs</td>
<td>https://archive.4plebs.org/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr> <tr>
<td>500px</td> <td>500px</td>
<td>https://500px.com/</td> <td>https://500px.com/</td>
@ -73,42 +67,12 @@ Consider all sites to be NSFW unless otherwise known.
<td>Galleries</td> <td>Galleries</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Allgirlbooru</td>
<td>https://allgirl.booru.org/</td>
<td>Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>arch.b4k.co</td>
<td>https://arch.b4k.co/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Archive of Sins</td>
<td>https://archiveofsins.com/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Archived.Moe</td>
<td>https://archived.moe/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr> <tr>
<td>ArtStation</td> <td>ArtStation</td>
<td>https://www.artstation.com/</td> <td>https://www.artstation.com/</td>
<td>Albums, Artwork Listings, Challenges, individual Images, Likes, Search Results, User Profiles</td> <td>Albums, Artwork Listings, Challenges, individual Images, Likes, Search Results, User Profiles</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>baraag</td>
<td>https://baraag.net/</td>
<td>Images from Statuses, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr>
<tr> <tr>
<td>Behance</td> <td>Behance</td>
<td>https://www.behance.net/</td> <td>https://www.behance.net/</td>
@ -139,24 +103,12 @@ Consider all sites to be NSFW unless otherwise known.
<td>Galleries, Posts, Search Results</td> <td>Galleries, Posts, Search Results</td>
<td><a href="configuration.rst#extractorderpibooruapi-key">API Key</a></td> <td><a href="configuration.rst#extractorderpibooruapi-key">API Key</a></td>
</tr> </tr>
<tr>
<td>Desuarchive</td>
<td>https://desuarchive.org/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr> <tr>
<td>DeviantArt</td> <td>DeviantArt</td>
<td>https://www.deviantart.com/</td> <td>https://www.deviantart.com/</td>
<td>Collections, Deviations, Favorites, Folders, Galleries, Journals, Popular Images, Scraps, Sta.sh, User Profiles</td> <td>Collections, Deviations, Favorites, Folders, Galleries, Journals, Popular Images, Scraps, Sta.sh, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td> <td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr> </tr>
<tr>
<td>Doki Reader</td>
<td>https://kobato.hologfx.com/reader/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr> <tr>
<td>Dynasty Reader</td> <td>Dynasty Reader</td>
<td>https://dynasty-scans.com/</td> <td>https://dynasty-scans.com/</td>
@ -199,18 +151,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Chapters, Manga</td> <td>Chapters, Manga</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Fashion Nova</td>
<td>https://www.fashionnova.com/</td>
<td>Collections, Products</td>
<td></td>
</tr>
<tr>
<td>Fireden</td>
<td>https://boards.fireden.net/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr> <tr>
<td>Flickr</td> <td>Flickr</td>
<td>https://www.flickr.com/</td> <td>https://www.flickr.com/</td>
@ -307,24 +247,12 @@ Consider all sites to be NSFW unless otherwise known.
<td>Galleries, Tag Searches</td> <td>Galleries, Tag Searches</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Hypnohub</td>
<td>https://hypnohub.net/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr> <tr>
<td>Idol Complex</td> <td>Idol Complex</td>
<td>https://idol.sankakucomplex.com/</td> <td>https://idol.sankakucomplex.com/</td>
<td>Pools, Posts, Tag Searches</td> <td>Pools, Posts, Tag Searches</td>
<td>Supported</td> <td>Supported</td>
</tr> </tr>
<tr>
<td>Illusioncardsbooru</td>
<td>https://illusioncards.booru.org/</td>
<td>Posts, Tag Searches</td>
<td></td>
</tr>
<tr> <tr>
<td>ImageBam</td> <td>ImageBam</td>
<td>http://www.imagebam.com/</td> <td>http://www.imagebam.com/</td>
@ -439,24 +367,12 @@ Consider all sites to be NSFW unless otherwise known.
<td>Soundtracks</td> <td>Soundtracks</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Kirei Cake</td>
<td>https://reader.kireicake.com/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr> <tr>
<td>Komikcast</td> <td>Komikcast</td>
<td>https://komikcast.com/</td> <td>https://komikcast.com/</td>
<td>Chapters, Manga</td> <td>Chapters, Manga</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Konachan</td>
<td>https://konachan.com/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr> <tr>
<td>LINE BLOG</td> <td>LINE BLOG</td>
<td>https://www.lineblog.me/</td> <td>https://www.lineblog.me/</td>
@ -469,12 +385,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Blogs, Posts</td> <td>Blogs, Posts</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Lolibooru</td>
<td>https://lolibooru.moe/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr> <tr>
<td>Luscious</td> <td>Luscious</td>
<td>https://members.luscious.net/</td> <td>https://members.luscious.net/</td>
@ -529,12 +439,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Albums, Channels</td> <td>Albums, Channels</td>
<td>Supported</td> <td>Supported</td>
</tr> </tr>
<tr>
<td>mastodon.social</td>
<td>https://mastodon.social/</td>
<td>Images from Statuses, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr>
<tr> <tr>
<td>My Hentai Gallery</td> <td>My Hentai Gallery</td>
<td>https://myhentaigallery.com/</td> <td>https://myhentaigallery.com/</td>
@ -595,24 +499,12 @@ Consider all sites to be NSFW unless otherwise known.
<td>Albums</td> <td>Albums</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Nyafuu Archive</td>
<td>https://archive.nyafuu.org/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr> <tr>
<td>Patreon</td> <td>Patreon</td>
<td>https://www.patreon.com/</td> <td>https://www.patreon.com/</td>
<td>Creators, Posts, User Profiles</td> <td>Creators, Posts, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#cookies">Cookies</a></td> <td><a href="https://github.com/mikf/gallery-dl#cookies">Cookies</a></td>
</tr> </tr>
<tr>
<td>Pawoo</td>
<td>https://pawoo.net/</td>
<td>Images from Statuses, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr>
<tr> <tr>
<td>Photobucket</td> <td>Photobucket</td>
<td>https://photobucket.com/</td> <td>https://photobucket.com/</td>
@ -685,12 +577,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>individual Images</td> <td>individual Images</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>PowerManga</td>
<td>https://read.powermanga.org/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr> <tr>
<td>Pururin</td> <td>Pururin</td>
<td>https://pururin.io/</td> <td>https://pururin.io/</td>
@ -703,18 +589,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Comic Issues, Comics</td> <td>Comic Issues, Comics</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Realbooru</td>
<td>https://realbooru.com/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>RebeccaBlackTech</td>
<td>https://rbt.asia/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr> <tr>
<td>Reddit</td> <td>Reddit</td>
<td>https://www.reddit.com/</td> <td>https://www.reddit.com/</td>
@ -733,24 +607,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Posts, Tag Searches</td> <td>Posts, Tag Searches</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Rule 34</td>
<td>https://rule34.xxx/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Safebooru</td>
<td>https://safebooru.org/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Sakugabooru</td>
<td>https://www.sakugabooru.com/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr> <tr>
<td>Sankaku Channel</td> <td>Sankaku Channel</td>
<td>https://sankaku.app/</td> <td>https://sankaku.app/</td>
@ -769,12 +625,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Chapters</td> <td>Chapters</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Sense-Scans</td>
<td>https://sensescans.com/reader/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr> <tr>
<td>Sex.com</td> <td>Sex.com</td>
<td>https://www.sex.com/</td> <td>https://www.sex.com/</td>
@ -817,24 +667,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Posts, User Profiles</td> <td>Posts, User Profiles</td>
<td>Supported</td> <td>Supported</td>
</tr> </tr>
<tr>
<td>Tbib</td>
<td>https://tbib.org/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>The /b/ Archive</td>
<td>https://thebarchive.com/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Thecollection</td>
<td>https://the-collection.booru.org/</td>
<td>Posts, Tag Searches</td>
<td></td>
</tr>
<tr> <tr>
<td>Tsumino</td> <td>Tsumino</td>
<td>https://www.tsumino.com/</td> <td>https://www.tsumino.com/</td>
@ -931,12 +763,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Galleries, User Profiles</td> <td>Galleries, User Profiles</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td>Yandere</td>
<td>https://yande.re/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr> <tr>
<td>yuki.la 4chan archive</td> <td>yuki.la 4chan archive</td>
<td>https://yuki.la/</td> <td>https://yuki.la/</td>
@ -961,5 +787,207 @@ Consider all sites to be NSFW unless otherwise known.
<td>Posts, User Profiles</td> <td>Posts, User Profiles</td>
<td></td> <td></td>
</tr> </tr>
<tr>
<td colspan="4"><strong>Gelbooru Beta 0.1.11</strong></td>
</tr>
<tr>
<td>Allgirlbooru</td>
<td>https://allgirl.booru.org/</td>
<td>Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Illusion Game Cards</td>
<td>https://illusioncards.booru.org/</td>
<td>Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>The /co/llection</td>
<td>https://the-collection.booru.org/</td>
<td>Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td colspan="4"><strong>Gelbooru Beta 0.2</strong></td>
</tr>
<tr>
<td>Realbooru</td>
<td>https://realbooru.com/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Rule 34</td>
<td>https://rule34.xxx/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Safebooru</td>
<td>https://safebooru.org/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>The Big ImageBoard</td>
<td>https://tbib.org/</td>
<td>Pools, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td colspan="4"><strong>Moebooru and MyImouto</strong></td>
</tr>
<tr>
<td>Hypnohub</td>
<td>https://hypnohub.net/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Konachan</td>
<td>https://konachan.com/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Lolibooru</td>
<td>https://lolibooru.moe/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>Sakugabooru</td>
<td>https://www.sakugabooru.com/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td>yande.re</td>
<td>https://yande.re/</td>
<td>Pools, Popular Images, Posts, Tag Searches</td>
<td></td>
</tr>
<tr>
<td colspan="4"><strong>FoolFuuka 4chan Archives</strong></td>
</tr>
<tr>
<td>4plebs</td>
<td>https://archive.4plebs.org/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>arch.b4k.co</td>
<td>https://arch.b4k.co/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Archive of Sins</td>
<td>https://archiveofsins.com/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Archived.Moe</td>
<td>https://archived.moe/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Desuarchive</td>
<td>https://desuarchive.org/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Fireden</td>
<td>https://boards.fireden.net/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>Nyafuu Archive</td>
<td>https://archive.nyafuu.org/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>RebeccaBlackTech</td>
<td>https://rbt.asia/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td>The /b/ Archive</td>
<td>https://thebarchive.com/</td>
<td>Boards, Search Results, Threads</td>
<td></td>
</tr>
<tr>
<td colspan="4"><strong>FoOlSlide Instances</strong></td>
</tr>
<tr>
<td>Doki Reader</td>
<td>https://kobato.hologfx.com/reader/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr>
<td>Kirei Cake</td>
<td>https://reader.kireicake.com/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr>
<td>PowerManga</td>
<td>https://read.powermanga.org/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr>
<td>Sense-Scans</td>
<td>https://sensescans.com/reader/</td>
<td>Chapters, Manga</td>
<td></td>
</tr>
<tr>
<td colspan="4"><strong>Mastodon Instances</strong></td>
</tr>
<tr>
<td>baraag</td>
<td>https://baraag.net/</td>
<td>Images from Statuses, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr>
<tr>
<td>mastodon.social</td>
<td>https://mastodon.social/</td>
<td>Images from Statuses, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr>
<tr>
<td>Pawoo</td>
<td>https://pawoo.net/</td>
<td>Images from Statuses, User Profiles</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr>
<tr>
<td colspan="4"><strong>Shopify Instances</strong></td>
</tr>
<tr>
<td>Fashion Nova</td>
<td>https://www.fashionnova.com/</td>
<td>Collections, Products</td>
<td></td>
</tr>
</tbody> </tbody>
</table> </table>

View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""Generate a reStructuredText document with all supported sites""" """Generate a Markdown document listing all supported sites"""
import os import os
import sys import sys
@ -15,6 +15,7 @@ CATEGORY_MAP = {
"2chan" : "Futaba Channel", "2chan" : "Futaba Channel",
"35photo" : "35PHOTO", "35photo" : "35PHOTO",
"adultempire" : "Adult Empire", "adultempire" : "Adult Empire",
"allgirl" : "All girl",
"archivedmoe" : "Archived.Moe", "archivedmoe" : "Archived.Moe",
"archiveofsins" : "Archive of Sins", "archiveofsins" : "Archive of Sins",
"artstation" : "ArtStation", "artstation" : "ArtStation",
@ -42,6 +43,7 @@ CATEGORY_MAP = {
"hentaihere" : "HentaiHere", "hentaihere" : "HentaiHere",
"hitomi" : "Hitomi.la", "hitomi" : "Hitomi.la",
"idolcomplex" : "Idol Complex", "idolcomplex" : "Idol Complex",
"illusioncardsbooru": "Illusion Game Cards",
"imagebam" : "ImageBam", "imagebam" : "ImageBam",
"imagefap" : "ImageFap", "imagefap" : "ImageFap",
"imgbb" : "ImgBB", "imgbb" : "ImgBB",
@ -90,7 +92,9 @@ CATEGORY_MAP = {
"smugmug" : "SmugMug", "smugmug" : "SmugMug",
"speakerdeck" : "Speaker Deck", "speakerdeck" : "Speaker Deck",
"subscribestar" : "SubscribeStar", "subscribestar" : "SubscribeStar",
"tbib" : "The Big ImageBoard",
"thebarchive" : "The /b/ Archive", "thebarchive" : "The /b/ Archive",
"thecollection" : "The /co/llection",
"tumblrgallery" : "TumblrGallery", "tumblrgallery" : "TumblrGallery",
"vanillarock" : "もえぴりあ", "vanillarock" : "もえぴりあ",
"vsco" : "VSCO", "vsco" : "VSCO",
@ -99,6 +103,7 @@ CATEGORY_MAP = {
"worldthree" : "World Three", "worldthree" : "World Three",
"xhamster" : "xHamster", "xhamster" : "xHamster",
"xvideos" : "XVideos", "xvideos" : "XVideos",
"yandere" : "yande.re",
"yuki" : "yuki.la 4chan archive", "yuki" : "yuki.la 4chan archive",
} }
@ -167,6 +172,14 @@ SUBCATEGORY_MAP = {
}, },
} }
BASE_MAP = {
"foolfuuka" : "FoolFuuka 4chan Archives",
"foolslide" : "FoOlSlide Instances",
"gelbooru_v01": "Gelbooru Beta 0.1.11",
"gelbooru_v02": "Gelbooru Beta 0.2",
"moebooru" : "Moebooru and MyImouto",
}
_OAUTH = '<a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a>' _OAUTH = '<a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a>'
_COOKIES = '<a href="https://github.com/mikf/gallery-dl#cookies">Cookies</a>' _COOKIES = '<a href="https://github.com/mikf/gallery-dl#cookies">Cookies</a>'
_APIKEY_DB = \ _APIKEY_DB = \
@ -275,7 +288,8 @@ def subcategory_key(sc):
def build_extractor_list(): def build_extractor_list():
"""Generate a sorted list of lists of extractor classes""" """Generate a sorted list of lists of extractor classes"""
categories = collections.defaultdict(list) categories = collections.defaultdict(lambda: collections.defaultdict(list))
default = categories[""]
domains = {} domains = {}
for extr in extractor._list_classes(): for extr in extractor._list_classes():
@ -283,21 +297,23 @@ def build_extractor_list():
if category in IGNORE_LIST: if category in IGNORE_LIST:
continue continue
if category: if category:
categories[category].append(extr.subcategory) default[category].append(extr.subcategory)
if category not in domains: if category not in domains:
domains[category] = domain(extr) domains[category] = domain(extr)
else: else:
base = categories[extr.basecategory]
for category, root in extr.instances: for category, root in extr.instances:
categories[category].append(extr.subcategory) base[category].append(extr.subcategory)
if category not in domains: if category not in domains:
domains[category] = root + "/" domains[category] = root + "/"
# sort subcategory lists # sort subcategory lists
for subcategories in categories.values(): for base in categories.values():
for subcategories in base.values():
subcategories.sort(key=subcategory_key) subcategories.sort(key=subcategory_key)
# add e-hentai.org # add e-hentai.org
categories["e-hentai"] = categories["exhentai"] default["e-hentai"] = default["exhentai"]
domains["e-hentai"] = domains["exhentai"].replace("x", "-") domains["e-hentai"] = domains["exhentai"].replace("x", "-")
return categories, domains return categories, domains
@ -328,7 +344,15 @@ def generate_output(columns, categories, domains):
tbody = [] tbody = []
append = tbody.append append = tbody.append
clist = sorted(categories.items(), key=category_key)
for name, base in categories.items():
if name and base:
name = BASE_MAP.get(name) or (name.capitalize() + " Instances")
append('\n<tr>\n <td colspan="4"><strong>' +
name + '</strong></td>\n</tr>')
clist = sorted(base.items(), key=category_key)
for category, subcategories in clist: for category, subcategories in clist:
append("<tr>") append("<tr>")
for column in columns: for column in columns: