[everia] update

- implement general _pagination method
- simplify code
- adjust URL patterns
- update test results

docs/supportedsites.md

@@ -230,7 +230,7 @@ Consider all listed sites to potentially be NSFW.
     <td></td>
 </tr>
 <tr>
-    <td>Everia</td>
+    <td>EVERIA.CLUB</td>
     <td>https://everia.club</td>
     <td>Categories, Dates, Posts, Search Results, Tag Searches</td>
     <td></td>

gallery_dl/extractor/everia.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2019-2023 Mike Fährmann
-#
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
@@ -15,92 +13,87 @@ import re
 BASE_PATTERN = r"(?:https?://)?everia\.club"
 
 
-class EveriaPostExtractor(Extractor):
+class EveriaExtractor(Extractor):
     category = "everia"
-    subcategory = "post"
     root = "https://everia.club"
-    pattern = BASE_PATTERN + r"/(\d{4}/\d{2}/\d{2}/[^/]+)/?"
-    example = "https://everia.club/0000/00/00/TITLE"
-    directory_fmt = ("{category}", "{title}")
-
-    def __init__(self, match):
-        super().__init__(match)
-        self.url = match.group(0)
 
     def items(self):
-        page = self.request(self.url).text
+        data = {"_extractor": EveriaPostExtractor}
+        for url in self.posts():
+            yield Message.Queue, url, data
+
+    def posts(self):
+        return self._pagination(self.groups[0])
+
+    def _pagination(self, path, params=None, pnum=1):
+        find_posts = re.compile(r'thumbnail">\s*<a href="([^"]+)').findall
+
+        while True:
+            if pnum == 1:
+                url = "{}{}/".format(self.root, path)
+            else:
+                url = "{}{}/page/{}/".format(self.root, path, pnum)
+            response = self.request(url, params=params, allow_redirects=False)
+
+            if response.status_code >= 300:
+                return
+
+            yield from find_posts(response.text)
+            pnum += 1
+
+
+class EveriaPostExtractor(EveriaExtractor):
+    subcategory = "post"
+    directory_fmt = ("{category}", "{title}")
+    archive_fmt = "{post_url}_{num}"
+    pattern = BASE_PATTERN + r"(/\d{4}/\d{2}/\d{2}/[^/?#]+)"
+    example = "https://everia.club/0000/00/00/TITLE"
+
+    def items(self):
+        url = self.root + self.groups[0]
+        page = self.request(url).text
         content = text.extr(page, 'itemprop="text">', "</div>")
-        urls = re.findall(r'img.*?src=\"(.+?)\"', content)
+        urls = re.findall(r'img.*?src="([^"]+)', content)
 
         data = {
             "title": text.unescape(
-                text.extr(page, 'itemprop="headline">', "</h1>")
+                text.extr(page, 'itemprop="headline">', "</h1>")),
-            ),
-            "url": self.url,
             "tags": list(text.extract_iter(page, 'rel="tag">', "</a>")),
+            "post_url": url,
             "post_category": text.extr(
-                page, "post-in-category-", " "
+                page, "post-in-category-", " ").capitalize(),
-            ).capitalize(),
             "count": len(urls),
         }
 
         yield Message.Directory, data
         for data["num"], url in enumerate(urls, 1):
-            text.nameext_from_url(text.unquote(url), data)
+            yield Message.Url, url, text.nameext_from_url(url, data)
-            yield Message.Url, url, data
 
 
-class EveriaTagExtractor(EveriaPostExtractor):
+class EveriaTagExtractor(EveriaExtractor):
     subcategory = "tag"
-    pattern = BASE_PATTERN + r"/(tag/[^/]+)/?"
+    pattern = BASE_PATTERN + r"(/tag/[^/?#]+)"
     example = "https://everia.club/tag/TAG"
 
-    def __init__(self, match):
-        super().__init__(match)
-        self.id = match.group(1)
 
-    def _posts(self, page):
+class EveriaCategoryExtractor(EveriaExtractor):
-        posts = re.findall(r'thumbnail\">\s*<a href=\"(.+?)\"', page)
-        for post in posts:
-            yield Message.Queue, post, {"_extractor": EveriaPostExtractor}
-
-    def items(self):
-        url = "{}/{}/".format(self.root, self.id)
-        page = self.request(url).text
-        yield from self._posts(page)
-        pages = list(text.extract_iter(page, "/page/", "/"))
-        if pages:
-            for i in range(2, int(pages[-2]) + 1):
-                url = "{}/{}/page/{}/".format(self.root, self.id, i + 1)
-                page = self.request(url).text
-                yield from self._posts(page)
-
-
-class EveriaSearchExtractor(EveriaTagExtractor):
-    subcategory = "search"
-    pattern = BASE_PATTERN + r"/(?:page/\d+/)?\?s=(.+)"
-    example = "https://everia.club/?s=SEARCH"
-
-    def items(self):
-        params = {"s": self.id}
-        page = self.request(self.root, params=params).text
-        yield from self._posts(page)
-        pages = list(text.extract_iter(page, "/page/", "/"))
-        if pages:
-            for i in range(2, int(pages[-2]) + 1):
-                url = "{}/page/{}/".format(self.root, i)
-                page = self.request(url, params=params).text
-                yield from self._posts(page)
-
-
-class EveriaCategoryExtractor(EveriaTagExtractor):
     subcategory = "category"
-    pattern = BASE_PATTERN + r"/(category/[^/]+)/?"
+    pattern = BASE_PATTERN + r"(/category/[^/?#]+)"
     example = "https://everia.club/category/CATEGORY"
 
 
-class EveriaDateExtractor(EveriaTagExtractor):
+class EveriaDateExtractor(EveriaExtractor):
     subcategory = "date"
-    pattern = BASE_PATTERN + \
+    pattern = (BASE_PATTERN +
-        r"/(\d{4}(?:/\d{2})?(?:/\d{2})?)(?:/page/\d+)?/?$"
+               r"(/\d{4}(?:/\d{2})?(?:/\d{2})?)(?:/page/\d+)?/?$")
     example = "https://everia.club/0000/00/00"
+
+
+class EveriaSearchExtractor(EveriaExtractor):
+    subcategory = "search"
+    pattern = BASE_PATTERN + r"/(?:page/\d+/)?\?s=([^&#]+)"
+    example = "https://everia.club/?s=SEARCH"
+
+    def posts(self):
+        params = {"s": self.groups[0]}
+        return self._pagination("", params)

scripts/supportedsites.py

@@ -49,6 +49,7 @@ CATEGORY_MAP = {
     "e926"           : "e926",
     "e6ai"           : "e6AI",
     "erome"          : "EroMe",
+    "everia"         : "EVERIA.CLUB",
     "e-hentai"       : "E-Hentai",
     "exhentai"       : "ExHentai",
     "fallenangels"   : "Fallen Angels Scans",

test/results/everia.py

@@ -9,29 +9,43 @@ from gallery_dl.extractor import everia
 
 __tests__ = (
 {
-    "#url"      : "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/",
+    "#url"  : "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/",
-    "#category" : ("", "everia", "post"),
+    "#class": everia.EveriaPostExtractor,
-    "#class"    : everia.EveriaPostExtractor,
+    "#count": 32,
-    "#archive"  : False,
-    "#count"    : 32,
 
-    "title"     : "Mikacho 조미카, JOApictures ‘Someday’",
+    "title"        : "Mikacho 조미카, JOApictures ‘Someday’",
     "post_category": "Korea",
-    "tags"      : ["[JOApictures]", "Mikacho 조미카"]
+    "tags"         : ["[JOApictures]", "Mikacho 조미카"]
 },
+
 {
-    "#url"      : "https://everia.club/tag/yeon-woo-연우/",
+    "#url"    : "https://everia.club/tag/miku-tanaka-%e7%94%b0%e4%b8%ad%e7%be%8e%e4%b9%85/",
-    "#category" : ("", "everia", "tag"),
+    "#class"  : everia.EveriaTagExtractor,
-    "#class"    : everia.EveriaTagExtractor,
+    "#pattern": everia.EveriaPostExtractor.pattern,
+    "#count"  : "> 50",
 },
+
 {
-    "#url"      : "https://everia.club/category/japan/",
+    "#url"    : "https://everia.club/category/japan/",
-    "#category" : ("", "everia", "category"),
+    "#class"  : everia.EveriaCategoryExtractor,
-    "#class"    : everia.EveriaCategoryExtractor,
+    "#pattern": everia.EveriaPostExtractor.pattern,
+    "#range"  : "1-50",
+    "#count"  : 50,
 },
+
 {
-    "#url"      : "https://everia.club/?s=saika",
+    "#url"    : "https://everia.club/2023/10/05/",
-    "#category" : ("", "everia", "search"),
+    "#class"  : everia.EveriaDateExtractor,
-    "#class"    : everia.EveriaSearchExtractor,
+    "#pattern": everia.EveriaPostExtractor.pattern,
-}
+    "#count"  : 34,
+},
+
+{
+    "#url"    : "https://everia.club/?s=saika",
+    "#class"  : everia.EveriaSearchExtractor,
+    "#pattern": everia.EveriaPostExtractor.pattern,
+    "#range"  : "1-15",
+    "#count"  : 15,
+},
+
 )