mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-22 02:32:33 +01:00
fix supportedsites.py
This commit is contained in:
Mike Fährmann
parent
5542a11c46
commit
03b219db9d
@ -15,8 +15,10 @@ Site URL Capabilities
|
||||
500px https://500px.com/ Galleries, individual Images, User Profiles
|
||||
8kun https://8kun.top/ Boards, Threads
|
||||
8muses https://comics.8muses.com/ Albums
|
||||
Acidimg https://acidimg.cc/ individual Images
|
||||
Adobe Portfolio https://www.myportfolio.com/ Galleries
|
||||
Adult Empire https://www.adultempire.com/ Galleries
|
||||
Allgirlbooru https://allgirl.booru.org/ Posts, Tag Searches
|
||||
arch.b4k.co https://arch.b4k.co/ Boards, Search Results, Threads
|
||||
Archive of Sins https://archiveofsins.com/ Boards, Search Results, Threads
|
||||
Archived.Moe https://archived.moe/ Boards, Search Results, Threads
|
||||
@ -56,17 +58,23 @@ Hiperdex https://hiperdex.com/ Artists, Chapters, Mang
|
||||
Hitomi.la https://hitomi.la/ Galleries, Tag Searches
|
||||
Hypnohub https://hypnohub.net/ Pools, Popular Images, Posts, Tag Searches
|
||||
Idol Complex https://idol.sankakucomplex.com/ Pools, Posts, Tag Searches Supported
|
||||
Illusioncardsbooru https://illusioncards.booru.org/ Posts, Tag Searches
|
||||
ImageBam http://www.imagebam.com/ Galleries, individual Images
|
||||
ImageChest https://imgchest.com/ Galleries
|
||||
ImageFap https://www.imagefap.com/ Galleries, individual Images, User Profiles
|
||||
Imagetwist https://imagetwist.com/ individual Images
|
||||
Imagevenue http://imagevenue.com/ individual Images
|
||||
ImgBB https://imgbb.com/ Albums, individual Images, User Profiles Supported
|
||||
imgbox https://imgbox.com/ Galleries, individual Images
|
||||
Imgclick https://imgclick.net/ individual Images
|
||||
Imgspice https://imgspice.com/ individual Images
|
||||
imgth https://imgth.com/ Galleries
|
||||
imgur https://imgur.com/ |imgur-C|
|
||||
Imxto https://imx.to/ individual Images
|
||||
Inkbunny https://inkbunny.net/ Posts, User Profiles Supported
|
||||
Instagram https://www.instagram.com/ |instagram-C| Supported
|
||||
Issuu https://issuu.com/ Publications, User Profiles
|
||||
Joyreactor http://joyreactor.com/ Posts, Search Results, Tag Searches, User Profiles
|
||||
Joyreactor http://joyreactor.cc/ Posts, Search Results, Tag Searches, User Profiles
|
||||
Keenspot http://www.keenspot.com/ Comics
|
||||
Kemonoparty https://kemono.party/ Posts, User Profiles
|
||||
Khinsider https://downloads.khinsider.com/ Soundtracks
|
||||
@ -103,11 +111,13 @@ PhotoVogue https://www.vogue.it/en/photovogue/ User Profiles
|
||||
Piczel https://piczel.tv/ Folders, individual Images, User Profiles
|
||||
Pillowfort https://www.pillowfort.social/ Posts, User Profiles
|
||||
Pinterest https://www.pinterest.com/ |pinterest-C| Supported
|
||||
Pixhost https://pixhost.to/ individual Images
|
||||
Pixiv https://www.pixiv.net/ |pixiv-C| `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
|
||||
Pixnet https://www.pixnet.net/ Folders, individual Images, Sets, User Profiles
|
||||
Plurk https://www.plurk.com/ Posts, Timelines
|
||||
Pornhub https://www.pornhub.com/ Galleries, User Profiles
|
||||
Pornreactor http://pornreactor.cc/ Posts, Search Results, Tag Searches, User Profiles
|
||||
Postimg https://postimages.org/ individual Images
|
||||
PowerManga https://read.powermanga.org/ Chapters, Manga
|
||||
Pururin https://pururin.io/ Galleries
|
||||
Read Comic Online https://readcomiconline.to/ Comic Issues, Comics
|
||||
@ -130,11 +140,15 @@ SlideShare https://www.slideshare.net/ Presentations
|
||||
SmugMug https://www.smugmug.com/ |smugmug-C| `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
|
||||
Speaker Deck https://speakerdeck.com/ Presentations
|
||||
SubscribeStar https://www.subscribestar.com/ Posts, User Profiles Supported
|
||||
Tbib https://tbib.org/ Pools, Posts, Tag Searches
|
||||
The /b/ Archive https://thebarchive.com/ Boards, Search Results, Threads
|
||||
Thecollection https://the-collection.booru.org/ Posts, Tag Searches
|
||||
Tsumino https://www.tsumino.com/ Galleries, Search Results Supported
|
||||
Tumblr https://www.tumblr.com/ Likes, Posts, Tag Searches, User Profiles `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
|
||||
Turboimagehost https://www.turboimagehost.com/ individual Images
|
||||
Twitter https://twitter.com/ |twitter-C| Supported
|
||||
Unsplash https://unsplash.com/ |unsplash-C|
|
||||
Vipr https://vipr.im/ individual Images
|
||||
VSCO https://vsco.co/ Collections, individual Images, User Profiles
|
||||
Wallhaven https://wallhaven.cc/ individual Images, Search Results `API Key <configuration.rst#extractorwallhavenapi-key>`__
|
||||
Warosu https://warosu.org/ Threads
|
||||
@ -146,16 +160,6 @@ xHamster https://xhamster.com/ Galleries, User Profile
|
||||
XVideos https://www.xvideos.com/ Galleries, User Profiles
|
||||
Yandere https://yande.re/ Pools, Popular Images, Posts, Tag Searches
|
||||
|yuki-S| https://yuki.la/ Threads
|
||||
Acidimg https://acidimg.cc/ individual Images
|
||||
Imagetwist https://imagetwist.com/ individual Images
|
||||
Imagevenue http://imagevenue.com/ individual Images
|
||||
Imgclick https://imgclick.net/ individual Images
|
||||
Imgspice https://imgspice.com/ individual Images
|
||||
Imxto https://imx.to/ individual Images
|
||||
Pixhost https://pixhost.to/ individual Images
|
||||
Postimg https://postimages.org/ individual Images
|
||||
Turboimagehost https://www.turboimagehost.com/ individual Images
|
||||
Vipr https://vipr.im/ individual Images
|
||||
かべうち https://kabe-uchiroom.com/ User Profiles
|
||||
もえぴりあ https://vanilla-rock.com/ Posts, Tag Searches
|
||||
半次元 https://bcy.net/ Posts, User Profiles
|
||||
|
||||
@ -232,16 +232,13 @@ def domain(cls):
|
||||
return ""
|
||||
|
||||
|
||||
def category_text(cls):
|
||||
def category_text(c):
|
||||
"""Return a human-readable representation of a category"""
|
||||
c = cls.category
|
||||
return CATEGORY_MAP.get(c) or c.capitalize()
|
||||
|
||||
|
||||
def subcategory_text(cls):
|
||||
def subcategory_text(c, sc):
|
||||
"""Return a human-readable representation of a subcategory"""
|
||||
c, sc = cls.category, cls.subcategory
|
||||
|
||||
if c in SUBCATEGORY_MAP:
|
||||
scm = SUBCATEGORY_MAP[c]
|
||||
if sc in scm:
|
||||
@ -254,66 +251,61 @@ def subcategory_text(cls):
|
||||
return sc if sc.endswith("s") else sc + "s"
|
||||
|
||||
|
||||
def category_key(cls):
|
||||
def category_key(c):
|
||||
"""Generate sorting keys by category"""
|
||||
key = category_text(cls).lower()
|
||||
if cls.__module__.endswith(".imagehosts"):
|
||||
key = "zz" + key
|
||||
return key
|
||||
return category_text(c[0]).lower()
|
||||
|
||||
|
||||
def subcategory_key(cls):
|
||||
def subcategory_key(sc):
|
||||
"""Generate sorting keys by subcategory"""
|
||||
if cls.subcategory == "issue":
|
||||
return "A"
|
||||
return cls.subcategory
|
||||
return "A" if sc == "issue" else sc
|
||||
|
||||
|
||||
def build_extractor_list():
|
||||
"""Generate a sorted list of lists of extractor classes"""
|
||||
extractors = collections.defaultdict(list)
|
||||
categories = collections.defaultdict(list)
|
||||
domains = {}
|
||||
|
||||
# get lists of extractor classes grouped by category
|
||||
for extr in extractor.extractors():
|
||||
if not extr.category or extr.category in IGNORE_LIST:
|
||||
for extr in extractor._list_classes():
|
||||
category = extr.category
|
||||
if category in IGNORE_LIST:
|
||||
continue
|
||||
extractors[extr.category].append(extr)
|
||||
if category:
|
||||
categories[category].append(extr.subcategory)
|
||||
if category not in domains:
|
||||
domains[category] = domain(extr)
|
||||
else:
|
||||
for category, root in extr.instances:
|
||||
categories[category].append(extr.subcategory)
|
||||
if category not in domains:
|
||||
domains[category] = root + "/"
|
||||
|
||||
# sort extractor lists with the same category
|
||||
for extrlist in extractors.values():
|
||||
extrlist.sort(key=subcategory_key)
|
||||
# sort subcategory lists
|
||||
for subcategories in categories.values():
|
||||
subcategories.sort(key=subcategory_key)
|
||||
|
||||
# ugly hack to add e-hentai.org
|
||||
eh = []
|
||||
for extr in extractors["exhentai"]:
|
||||
class eh_extr(extr):
|
||||
category = "e-hentai"
|
||||
root = "https://e-hentai.org"
|
||||
eh.append(eh_extr)
|
||||
extractors["e-hentai"] = eh
|
||||
# add e-hentai.org
|
||||
categories["e-hentai"] = categories["exhentai"]
|
||||
domains["e-hentai"] = domains["exhentai"].replace("x", "-")
|
||||
|
||||
# sort lists by category
|
||||
return sorted(
|
||||
extractors.values(),
|
||||
key=lambda lst: category_key(lst[0]),
|
||||
)
|
||||
return categories, domains
|
||||
|
||||
|
||||
# define table columns
|
||||
COLUMNS = (
|
||||
("Site", 20,
|
||||
lambda x: category_text(x[0])),
|
||||
lambda c, scs, d: category_text(c)),
|
||||
("URL" , 35,
|
||||
lambda x: domain(x[0])),
|
||||
lambda c, scs, d: d),
|
||||
("Capabilities", 50,
|
||||
lambda x: ", ".join(subcategory_text(extr) for extr in x
|
||||
if subcategory_text(extr))),
|
||||
lambda c, scs, d: ", ".join(subcategory_text(c, sc) for sc in scs
|
||||
if subcategory_text(c, sc))),
|
||||
("Authentication", 16,
|
||||
lambda x: AUTH_MAP.get(x[0].category, "")),
|
||||
lambda c, scs, d: AUTH_MAP.get(c, "")),
|
||||
)
|
||||
|
||||
|
||||
def write_output(fobj, columns, extractors):
|
||||
def write_output(fp, columns, categories, domains):
|
||||
|
||||
def pad(output, col, category=None):
|
||||
size = col[1]
|
||||
@ -326,7 +318,7 @@ def write_output(fobj, columns, extractors):
|
||||
|
||||
return output + " " * (size - len(output))
|
||||
|
||||
w = fobj.write
|
||||
w = fp.write
|
||||
subs = []
|
||||
|
||||
# caption
|
||||
@ -343,9 +335,11 @@ def write_output(fobj, columns, extractors):
|
||||
w(sep)
|
||||
|
||||
# table body
|
||||
for lst in extractors:
|
||||
clist = sorted(categories.items(), key=category_key)
|
||||
for category, subcategories in clist:
|
||||
domain = domains[category]
|
||||
w(" ".join(
|
||||
pad(col[2](lst), col, lst[0].category)
|
||||
pad(col[2](category, subcategories, domain), col, category)
|
||||
for col in columns
|
||||
).strip())
|
||||
w("\n")
|
||||
@ -359,6 +353,7 @@ def write_output(fobj, columns, extractors):
|
||||
w(".. {} replace:: {}\n".format(sub, value))
|
||||
|
||||
|
||||
categories, domains = build_extractor_list()
|
||||
outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst"
|
||||
with open(util.path("docs", outfile), "w") as file:
|
||||
write_output(file, COLUMNS, build_extractor_list())
|
||||
write_output(file, COLUMNS, categories, domains)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user