[civitai] add 'uuid' metadata field & use as archive format (#6326)

gallery_dl/extractor/civitai.py

@@ -23,7 +23,7 @@ class CivitaiExtractor(Extractor):
     root = "https://civitai.com"
     directory_fmt = ("{category}", "{username|user[username]}", "images")
     filename_fmt = "{file[id]|id|filename}.{extension}"
-    archive_fmt = "{file[hash]|hash}"
+    archive_fmt = "{file[uuid]|uuid}"
     request_interval = (0.5, 1.5)
 
     def _init(self):
@@ -101,9 +101,11 @@ class CivitaiExtractor(Extractor):
     def _url(self, image):
         url = image["url"]
         if "/" in url:
-            parts = url.rsplit("/", 2)
-            parts[1] = self._image_quality
+            parts = url.rsplit("/", 3)
+            image["uuid"] = parts[1]
+            parts[2] = self._image_quality
             return "/".join(parts)
+        image["uuid"] = url
 
         name = image.get("name")
         if not name:
@@ -133,8 +135,6 @@ class CivitaiModelExtractor(CivitaiExtractor):
     directory_fmt = ("{category}", "{user[username]}",
                      "{model[id]}{model[name]:? //}",
                      "{version[id]}{version[name]:? //}")
-    filename_fmt = "{file[id]}.{extension}"
-    archive_fmt = "{file[hash]}"
     pattern = BASE_PATTERN + r"/models/(\d+)(?:/?\?modelVersionId=(\d+))?"
     example = "https://civitai.com/models/12345/TITLE"
 
@@ -195,8 +195,12 @@ class CivitaiModelExtractor(CivitaiExtractor):
         )
 
     def _extract_files_model(self, model, version, user):
-        return [
-            {
+        files = []
+
+        for num, file in enumerate(version["files"], 1):
+            file["uuid"] = "model-{}-{}-{}".format(
+                model["id"], version["id"], file["id"])
+            files.append({
                 "num"      : num,
                 "file"     : file,
                 "filename" : file["name"],
@@ -207,9 +211,9 @@ class CivitaiModelExtractor(CivitaiExtractor):
                 "_http_headers" : {
                     "Authorization": self.api.headers.get("Authorization")},
                 "_http_validate": self._validate_file_model,
-            }
-            for num, file in enumerate(version["files"], 1)
-        ]
+            })
+
+        return files
 
     def _extract_files_image(self, model, version, user):
         if "images" in version:

test/results/civitai.py

@@ -26,13 +26,15 @@ __tests__ = (
         "minor"      : False,
         "name"       : "メイド　クラシック/maid classic",
         "nsfwLevel"  : 1,
-        "type"       : "LORA"
+        "type"       : "LORA",
     },
     "user"   : {
         "image"   : None,
         "username": "bolero537"
     },
-    "file"   : dict,
+    "file"   : {
+        "uuid": str,
+    },
     "version": dict,
     "num"    : range(1, 3),
 },
@@ -60,7 +62,10 @@ __tests__ = (
         "username": "bolero537"
     },
     "file"   : {
-        "id": {26887862, 26887856, 26887852},
+        "id"  : {26887862, 26887856, 26887852},
+        "uuid": {"52b6efa7-801c-4901-90b4-fa3964d23480",
+                 "c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c",
+                 "68568d22-c4f3-45cb-ac32-82f1cedf968f"},
     },
     "model"  : {
         "id": 703211,
@@ -89,6 +94,7 @@ __tests__ = (
     "user"     : {
         "username": "bolero537",
     },
+    "uuid"     : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
     "width"    : 1152,
 },
 
@@ -105,6 +111,12 @@ __tests__ = (
         "id"  : 6877551,
         "date": "dt:2024-09-22 12:54:15",
     },
+    "file": {
+        "id"  : {30748752, 30748747, 30748733},
+        "uuid": {"6220fa0f-9037-4b1d-bfbd-a740a06eeb7c",
+                 "cd1edb7f-7b50-4da5-bf23-d38f24d8aef0",
+                 "cfd5b231-accd-49bd-8bde-370880f63aa6"},
+    },
 },
 
 {