[everia.club] Add support

- Unescape title and URL
- Add tags and categories metadata
    Lookup tag id with API instead of downloading tag page
- Add category extractor
- Add tests
- Rename EveriaExtractor to EveriaPostExtractor
- Fix EveriaPostExtractor example
- Lookup tags/categories by post id
- Add date extractor
- Remove leftover pages parameter
- Add error handling for invalid dates.
- Add filename numbering
    Parse date
- Rename extract() to images()
- Remove html import
- Fix search/date URLs with page number
- Fix tag/category search
- Fix post extractor
- Fix tag, category extractors
- Fix search extractor
- Only load first page once
- Fix date extractor
- Fix tests
- Clean up search extractor

docs/supportedsites.md

@@ -229,6 +229,12 @@ Consider all listed sites to potentially be NSFW.
     <td>Albums, Search Results, User Profiles</td>
     <td></td>
 </tr>
+<tr>
+    <td>Everia</td>
+    <td>https://everia.club</td>
+    <td>Categories, Dates, Posts, Search Results, Tag Searches</td>
+    <td></td>
+</tr>
 <tr>
     <td>ExHentai</td>
     <td>https://exhentai.org/</td>

gallery_dl/extractor/__init__.py

@@ -47,6 +47,7 @@ modules = [
     "dynastyscans",
     "e621",
     "erome",
+    "everia",
     "exhentai",
     "fanbox",
     "fanleaks",

gallery_dl/extractor/everia.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019-2023 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://everia.club"""
+
+from .common import Extractor, Message
+from .. import text
+import re
+
+BASE_PATTERN = r"(?:https?://)?everia\.club"
+
+
+class EveriaPostExtractor(Extractor):
+    category = "everia"
+    subcategory = "post"
+    root = "https://everia.club"
+    pattern = BASE_PATTERN + r"/(\d{4}/\d{2}/\d{2}/[^/]+)/?"
+    example = "https://everia.club/0000/00/00/TITLE"
+    directory_fmt = ("{category}", "{title}")
+
+    def __init__(self, match):
+        super().__init__(match)
+        self.url = match.group(0)
+
+    def items(self):
+        page = self.request(self.url).text
+        content = text.extr(page, 'itemprop="text">', "</div>")
+        urls = re.findall(r'img.*?src=\"(.+?)\"', content)
+
+        data = {
+            "title": text.unescape(
+                text.extr(page, 'itemprop="headline">', "</h1>")
+            ),
+            "url": self.url,
+            "tags": list(text.extract_iter(page, 'rel="tag">', "</a>")),
+            "post_category": text.extr(
+                page, "post-in-category-", " "
+            ).capitalize(),
+            "count": len(urls),
+        }
+
+        yield Message.Directory, data
+        for data["num"], url in enumerate(urls, 1):
+            text.nameext_from_url(text.unquote(url), data)
+            yield Message.Url, url, data
+
+
+class EveriaTagExtractor(EveriaPostExtractor):
+    subcategory = "tag"
+    pattern = BASE_PATTERN + r"/(tag/[^/]+)/?"
+    example = "https://everia.club/tag/TAG"
+
+    def __init__(self, match):
+        super().__init__(match)
+        self.id = match.group(1)
+
+    def _posts(self, page):
+        posts = re.findall(r'thumbnail\">\s*<a href=\"(.+?)\"', page)
+        for post in posts:
+            yield Message.Queue, post, {"_extractor": EveriaPostExtractor}
+
+    def items(self):
+        url = "{}/{}/".format(self.root, self.id)
+        page = self.request(url).text
+        yield from self._posts(page)
+        pages = list(text.extract_iter(page, "/page/", "/"))
+        if pages:
+            for i in range(2, int(pages[-2]) + 1):
+                url = "{}/{}/page/{}/".format(self.root, self.id, i + 1)
+                page = self.request(url).text
+                yield from self._posts(page)
+
+
+class EveriaSearchExtractor(EveriaTagExtractor):
+    subcategory = "search"
+    pattern = BASE_PATTERN + r"/(?:page/\d+/)?\?s=(.+)"
+    example = "https://everia.club/?s=SEARCH"
+
+    def items(self):
+        params = {"s": self.id}
+        page = self.request(self.root, params=params).text
+        yield from self._posts(page)
+        pages = list(text.extract_iter(page, "/page/", "/"))
+        if pages:
+            for i in range(2, int(pages[-2]) + 1):
+                url = "{}/page/{}/".format(self.root, i)
+                page = self.request(url, params=params).text
+                yield from self._posts(page)
+
+
+class EveriaCategoryExtractor(EveriaTagExtractor):
+    subcategory = "category"
+    pattern = BASE_PATTERN + r"/(category/[^/]+)/?"
+    example = "https://everia.club/category/CATEGORY"
+
+
+class EveriaDateExtractor(EveriaTagExtractor):
+    subcategory = "date"
+    pattern = BASE_PATTERN + \
+        r"/(\d{4}(?:/\d{2})?(?:/\d{2})?)(?:/page/\d+)?/?$"
+    example = "https://everia.club/0000/00/00"

test/results/everia.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import everia
+
+
+__tests__ = (
+{
+    "#url"      : "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/",
+    "#category" : ("", "everia", "post"),
+    "#class"    : everia.EveriaPostExtractor,
+    "#archive"  : False,
+    "#count"    : 32,
+
+    "title"     : "Mikacho 조미카, JOApictures ‘Someday’",
+    "post_category": "Korea",
+    "tags"      : ["[JOApictures]", "Mikacho 조미카"]
+},
+{
+    "#url"      : "https://everia.club/tag/yeon-woo-연우/",
+    "#category" : ("", "everia", "tag"),
+    "#class"    : everia.EveriaTagExtractor,
+},
+{
+    "#url"      : "https://everia.club/category/japan/",
+    "#category" : ("", "everia", "category"),
+    "#class"    : everia.EveriaCategoryExtractor,
+},
+{
+    "#url"      : "https://everia.club/?s=saika",
+    "#category" : ("", "everia", "search"),
+    "#class"    : everia.EveriaSearchExtractor,
+}
+)