[civitai] add extractors for global 'models' and 'images' (#6310)

docs/supportedsites.md

@@ -166,7 +166,7 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>Civitai</td>
     <td>https://www.civitai.com/</td>
-    <td>individual Images, Models, Posts, Search Results, Tag Searches (Images), Tag Searches (Models), User Profiles, User Images, User Models, User Posts</td>
+    <td>individual Images, Image Listings, Models, Model Listings, Posts, Search Results, Tag Searches, User Profiles, User Images, User Models, User Posts</td>
     <td></td>
 </tr>
 <tr>

gallery_dl/extractor/civitai.py

@@ -269,24 +269,14 @@ class CivitaiPostExtractor(CivitaiExtractor):
         return ({"id": int(self.groups[0])},)
 
 
-class CivitaiTagModelsExtractor(CivitaiExtractor):
-    subcategory = "tag-models"
-    pattern = BASE_PATTERN + r"/(?:tag/|models\?tag=)([^/?&#]+)"
+class CivitaiTagExtractor(CivitaiExtractor):
+    subcategory = "tag"
+    pattern = BASE_PATTERN + r"/tag/([^/?&#]+)"
     example = "https://civitai.com/tag/TAG"
 
     def models(self):
         tag = text.unquote(self.groups[0])
-        return self.api.models({"tag": tag})
-
-
-class CivitaiTagImagesExtractor(CivitaiExtractor):
-    subcategory = "tag-images"
-    pattern = BASE_PATTERN + r"/images\?tags=([^&#]+)"
-    example = "https://civitai.com/images?tags=12345"
-
-    def images(self):
-        tag = text.unquote(self.groups[0])
-        return self.api.images({"tag": tag})
+        return self.api.models_tag(tag)
 
 
 class CivitaiSearchExtractor(CivitaiExtractor):
@@ -299,6 +289,26 @@ class CivitaiSearchExtractor(CivitaiExtractor):
         return self.api.models(params)
 
 
+class CivitaiModelsExtractor(CivitaiExtractor):
+    subcategory = "models"
+    pattern = BASE_PATTERN + r"/models(?:/?\?([^#]+))?(?:$|#)"
+    example = "https://civitai.com/models"
+
+    def models(self):
+        params = text.parse_query(self.groups[0])
+        return self.api.models(params)
+
+
+class CivitaiImagesExtractor(CivitaiExtractor):
+    subcategory = "images"
+    pattern = BASE_PATTERN + r"/images(?:/?\?([^#]+))?(?:$|#)"
+    example = "https://civitai.com/images"
+
+    def images(self):
+        params = text.parse_query(self.groups[0])
+        return self.api.images(params)
+
+
 class CivitaiUserExtractor(CivitaiExtractor):
     subcategory = "user"
     pattern = USER_PATTERN + r"/?(?:$|\?|#)"
@@ -402,6 +412,9 @@ class CivitaiRestAPI():
     def models(self, params):
         return self._pagination("/v1/models", params)
 
+    def models_tag(self, tag):
+        return self.models({"tag": tag})
+
     def _call(self, endpoint, params=None):
         if endpoint[0] == "/":
             url = self.root + endpoint
@@ -469,6 +482,7 @@ class CivitaiTrpcAPI():
                 "include"      : ["cosmetics"],
             })
 
+        params = self._type_params(params)
         return self._pagination(endpoint, params)
 
     def images_gallery(self, model, version, user):
@@ -522,6 +536,9 @@ class CivitaiTrpcAPI():
 
         return self._pagination(endpoint, params)
 
+    def models_tag(self, tag):
+        return self.models({"tagname": tag})
+
     def post(self, post_id):
         endpoint = "post.get"
         params = {"id": int(post_id)}
@@ -586,3 +603,13 @@ class CivitaiTrpcAPI():
     def _merge_params(self, params_user, params_default):
         params_default.update(params_user)
         return params_default
+
+    def _type_params(self, params):
+        for key, type in (
+            ("tags"          , int),
+            ("modelId"       , int),
+            ("modelVersionId", int),
+        ):
+            if key in params:
+                params[key] = type(params[key])
+        return params

scripts/supportedsites.py

@@ -198,8 +198,8 @@ SUBCATEGORY_MAP = {
         "feed": "Subscriptions Feed",
     },
     "civitai": {
-        "tag-models": "Tag Searches (Models)",
-        "tag-images": "Tag Searches (Images)",
+        "models": "Model Listings",
+        "images": "Image Listings",
         "user-models": "User Models",
         "user-images": "User Images",
         "user-posts" : "User Posts",

test/results/civitai.py

@@ -121,12 +121,22 @@ __tests__ = (
 
 {
     "#url"  : "https://civitai.com/tag/mecha",
-    "#class": civitai.CivitaiTagModelsExtractor,
+    "#class": civitai.CivitaiTagExtractor,
 },
 
 {
     "#url"  : "https://civitai.com/images?tags=482",
-    "#class": civitai.CivitaiTagImagesExtractor,
+    "#class": civitai.CivitaiImagesExtractor,
+},
+
+{
+    "#url"  : "https://civitai.com/images?modelVersionId=786644",
+    "#class": civitai.CivitaiImagesExtractor,
+},
+
+{
+    "#url"  : "https://civitai.com/models",
+    "#class": civitai.CivitaiModelsExtractor,
 },
 
 {