diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index c2700ef1..9d3d08b5 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -15,8 +15,10 @@ Site URL Capabilities 500px https://500px.com/ Galleries, individual Images, User Profiles 8kun https://8kun.top/ Boards, Threads 8muses https://comics.8muses.com/ Albums +Acidimg https://acidimg.cc/ individual Images Adobe Portfolio https://www.myportfolio.com/ Galleries Adult Empire https://www.adultempire.com/ Galleries +Allgirlbooru https://allgirl.booru.org/ Posts, Tag Searches arch.b4k.co https://arch.b4k.co/ Boards, Search Results, Threads Archive of Sins https://archiveofsins.com/ Boards, Search Results, Threads Archived.Moe https://archived.moe/ Boards, Search Results, Threads @@ -56,17 +58,23 @@ Hiperdex https://hiperdex.com/ Artists, Chapters, Mang Hitomi.la https://hitomi.la/ Galleries, Tag Searches Hypnohub https://hypnohub.net/ Pools, Popular Images, Posts, Tag Searches Idol Complex https://idol.sankakucomplex.com/ Pools, Posts, Tag Searches Supported +Illusioncardsbooru https://illusioncards.booru.org/ Posts, Tag Searches ImageBam http://www.imagebam.com/ Galleries, individual Images ImageChest https://imgchest.com/ Galleries ImageFap https://www.imagefap.com/ Galleries, individual Images, User Profiles +Imagetwist https://imagetwist.com/ individual Images +Imagevenue http://imagevenue.com/ individual Images ImgBB https://imgbb.com/ Albums, individual Images, User Profiles Supported imgbox https://imgbox.com/ Galleries, individual Images +Imgclick https://imgclick.net/ individual Images +Imgspice https://imgspice.com/ individual Images imgth https://imgth.com/ Galleries imgur https://imgur.com/ |imgur-C| +Imxto https://imx.to/ individual Images Inkbunny https://inkbunny.net/ Posts, User Profiles Supported Instagram https://www.instagram.com/ |instagram-C| Supported Issuu https://issuu.com/ Publications, User Profiles -Joyreactor http://joyreactor.com/ Posts, Search Results, Tag Searches, User Profiles +Joyreactor http://joyreactor.cc/ Posts, Search Results, Tag Searches, User Profiles Keenspot http://www.keenspot.com/ Comics Kemonoparty https://kemono.party/ Posts, User Profiles Khinsider https://downloads.khinsider.com/ Soundtracks @@ -103,11 +111,13 @@ PhotoVogue https://www.vogue.it/en/photovogue/ User Profiles Piczel https://piczel.tv/ Folders, individual Images, User Profiles Pillowfort https://www.pillowfort.social/ Posts, User Profiles Pinterest https://www.pinterest.com/ |pinterest-C| Supported +Pixhost https://pixhost.to/ individual Images Pixiv https://www.pixiv.net/ |pixiv-C| `OAuth `__ Pixnet https://www.pixnet.net/ Folders, individual Images, Sets, User Profiles Plurk https://www.plurk.com/ Posts, Timelines Pornhub https://www.pornhub.com/ Galleries, User Profiles Pornreactor http://pornreactor.cc/ Posts, Search Results, Tag Searches, User Profiles +Postimg https://postimages.org/ individual Images PowerManga https://read.powermanga.org/ Chapters, Manga Pururin https://pururin.io/ Galleries Read Comic Online https://readcomiconline.to/ Comic Issues, Comics @@ -130,11 +140,15 @@ SlideShare https://www.slideshare.net/ Presentations SmugMug https://www.smugmug.com/ |smugmug-C| `OAuth `__ Speaker Deck https://speakerdeck.com/ Presentations SubscribeStar https://www.subscribestar.com/ Posts, User Profiles Supported +Tbib https://tbib.org/ Pools, Posts, Tag Searches The /b/ Archive https://thebarchive.com/ Boards, Search Results, Threads +Thecollection https://the-collection.booru.org/ Posts, Tag Searches Tsumino https://www.tsumino.com/ Galleries, Search Results Supported Tumblr https://www.tumblr.com/ Likes, Posts, Tag Searches, User Profiles `OAuth `__ +Turboimagehost https://www.turboimagehost.com/ individual Images Twitter https://twitter.com/ |twitter-C| Supported Unsplash https://unsplash.com/ |unsplash-C| +Vipr https://vipr.im/ individual Images VSCO https://vsco.co/ Collections, individual Images, User Profiles Wallhaven https://wallhaven.cc/ individual Images, Search Results `API Key `__ Warosu https://warosu.org/ Threads @@ -146,16 +160,6 @@ xHamster https://xhamster.com/ Galleries, User Profile XVideos https://www.xvideos.com/ Galleries, User Profiles Yandere https://yande.re/ Pools, Popular Images, Posts, Tag Searches |yuki-S| https://yuki.la/ Threads -Acidimg https://acidimg.cc/ individual Images -Imagetwist https://imagetwist.com/ individual Images -Imagevenue http://imagevenue.com/ individual Images -Imgclick https://imgclick.net/ individual Images -Imgspice https://imgspice.com/ individual Images -Imxto https://imx.to/ individual Images -Pixhost https://pixhost.to/ individual Images -Postimg https://postimages.org/ individual Images -Turboimagehost https://www.turboimagehost.com/ individual Images -Vipr https://vipr.im/ individual Images かべうち https://kabe-uchiroom.com/ User Profiles もえぴりあ https://vanilla-rock.com/ Posts, Tag Searches 半次元 https://bcy.net/ Posts, User Profiles diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 3a37095a..eec99f21 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -232,16 +232,13 @@ def domain(cls): return "" -def category_text(cls): +def category_text(c): """Return a human-readable representation of a category""" - c = cls.category return CATEGORY_MAP.get(c) or c.capitalize() -def subcategory_text(cls): +def subcategory_text(c, sc): """Return a human-readable representation of a subcategory""" - c, sc = cls.category, cls.subcategory - if c in SUBCATEGORY_MAP: scm = SUBCATEGORY_MAP[c] if sc in scm: @@ -254,66 +251,61 @@ def subcategory_text(cls): return sc if sc.endswith("s") else sc + "s" -def category_key(cls): +def category_key(c): """Generate sorting keys by category""" - key = category_text(cls).lower() - if cls.__module__.endswith(".imagehosts"): - key = "zz" + key - return key + return category_text(c[0]).lower() -def subcategory_key(cls): +def subcategory_key(sc): """Generate sorting keys by subcategory""" - if cls.subcategory == "issue": - return "A" - return cls.subcategory + return "A" if sc == "issue" else sc def build_extractor_list(): """Generate a sorted list of lists of extractor classes""" - extractors = collections.defaultdict(list) + categories = collections.defaultdict(list) + domains = {} - # get lists of extractor classes grouped by category - for extr in extractor.extractors(): - if not extr.category or extr.category in IGNORE_LIST: + for extr in extractor._list_classes(): + category = extr.category + if category in IGNORE_LIST: continue - extractors[extr.category].append(extr) + if category: + categories[category].append(extr.subcategory) + if category not in domains: + domains[category] = domain(extr) + else: + for category, root in extr.instances: + categories[category].append(extr.subcategory) + if category not in domains: + domains[category] = root + "/" - # sort extractor lists with the same category - for extrlist in extractors.values(): - extrlist.sort(key=subcategory_key) + # sort subcategory lists + for subcategories in categories.values(): + subcategories.sort(key=subcategory_key) - # ugly hack to add e-hentai.org - eh = [] - for extr in extractors["exhentai"]: - class eh_extr(extr): - category = "e-hentai" - root = "https://e-hentai.org" - eh.append(eh_extr) - extractors["e-hentai"] = eh + # add e-hentai.org + categories["e-hentai"] = categories["exhentai"] + domains["e-hentai"] = domains["exhentai"].replace("x", "-") - # sort lists by category - return sorted( - extractors.values(), - key=lambda lst: category_key(lst[0]), - ) + return categories, domains # define table columns COLUMNS = ( ("Site", 20, - lambda x: category_text(x[0])), + lambda c, scs, d: category_text(c)), ("URL" , 35, - lambda x: domain(x[0])), + lambda c, scs, d: d), ("Capabilities", 50, - lambda x: ", ".join(subcategory_text(extr) for extr in x - if subcategory_text(extr))), + lambda c, scs, d: ", ".join(subcategory_text(c, sc) for sc in scs + if subcategory_text(c, sc))), ("Authentication", 16, - lambda x: AUTH_MAP.get(x[0].category, "")), + lambda c, scs, d: AUTH_MAP.get(c, "")), ) -def write_output(fobj, columns, extractors): +def write_output(fp, columns, categories, domains): def pad(output, col, category=None): size = col[1] @@ -326,7 +318,7 @@ def write_output(fobj, columns, extractors): return output + " " * (size - len(output)) - w = fobj.write + w = fp.write subs = [] # caption @@ -343,9 +335,11 @@ def write_output(fobj, columns, extractors): w(sep) # table body - for lst in extractors: + clist = sorted(categories.items(), key=category_key) + for category, subcategories in clist: + domain = domains[category] w(" ".join( - pad(col[2](lst), col, lst[0].category) + pad(col[2](category, subcategories, domain), col, category) for col in columns ).strip()) w("\n") @@ -359,6 +353,7 @@ def write_output(fobj, columns, extractors): w(".. {} replace:: {}\n".format(sub, value)) +categories, domains = build_extractor_list() outfile = sys.argv[1] if len(sys.argv) > 1 else "supportedsites.rst" with open(util.path("docs", outfile), "w") as file: - write_output(file, COLUMNS, build_extractor_list()) + write_output(file, COLUMNS, categories, domains)