diff --git a/docs/configuration.rst b/docs/configuration.rst index e08ad53e..e72f9bff 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -387,8 +387,8 @@ Default ``[Danbooru]``, ``[E621]``, ``[foolfuuka]:search``, ``itaku``, ``koharu``, ``newgrounds``, ``[philomena]``, ``pixiv:novel``, ``plurk``, - ``poipiku`` , ``pornpics``, ``soundgasm``, ``urlgalleries``, - ``vk``, ``zerochan`` + ``poipiku`` , ``pornpics``, ``scrolller``, ``soundgasm``, + ``urlgalleries``, ``vk``, ``zerochan`` * ``"1.0-2.0"`` ``flickr``, ``weibo``, ``[wikimedia]`` * ``"2.0-4.0"`` @@ -1656,6 +1656,23 @@ Description It is possible to use ``"all"`` instead of listing all values separately. +extractor.civitai.metadata +-------------------------- +Type + * ``bool`` + * ``string`` + * ``list`` of ``strings`` +Default + ``false`` +Example + * ``"generation"`` + * ``["generation"]`` +Description + Extract additional ``generation`` metadata. + + Note: This requires 1 additional HTTP request per image. + + extractor.civitai.nsfw ---------------------- Type @@ -1788,7 +1805,7 @@ Type Default ``false`` Example - * ``replacements,comments,ai_tags`` + * ``"replacements,comments,ai_tags"`` * ``["replacements", "comments", "ai_tags"]`` Description Extract additional metadata diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index 0b1e44af..1e8cb424 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -44,6 +44,16 @@ class CivitaiExtractor(Extractor): self._image_quality = "original=true" self._image_ext = "png" + metadata = self.config("metadata") + if metadata: + if isinstance(metadata, str): + metadata = metadata.split(",") + elif not isinstance(metadata, (list, tuple)): + metadata = ("generation",) + self._meta_generation = ("generation" in metadata) + else: + self._meta_generation = False + def items(self): models = self.models() if models: @@ -81,6 +91,9 @@ class CivitaiExtractor(Extractor): if images: for image in images: url = self._url(image) + if self._meta_generation: + image["generation"] = self.api.image_generationdata( + image["id"]) image["date"] = text.parse_datetime( image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") text.nameext_from_url(url, image) @@ -127,6 +140,8 @@ class CivitaiExtractor(Extractor): data["extension"] = self._image_ext if "id" not in file and data["filename"].isdecimal(): file["id"] = text.parse_int(data["filename"]) + if self._meta_generation: + file["generation"] = self.api.image_generationdata(file["id"]) yield data @@ -469,7 +484,7 @@ class CivitaiTrpcAPI(): self.root = extractor.root + "/api/trpc/" self.headers = { "content-type" : "application/json", - "x-client-version": "5.0.185", + "x-client-version": "5.0.211", "x-client-date" : "", "x-client" : "web", "x-fingerprint" : "undefined", @@ -491,6 +506,11 @@ class CivitaiTrpcAPI(): params = {"id": int(image_id)} return (self._call(endpoint, params),) + def image_generationdata(self, image_id): + endpoint = "image.getGenerationData" + params = {"id": int(image_id)} + return self._call(endpoint, params) + def images(self, params, defaults=True): endpoint = "image.getInfinite" diff --git a/test/results/civitai.py b/test/results/civitai.py index 716d5ecd..24976749 100644 --- a/test/results/civitai.py +++ b/test/results/civitai.py @@ -77,7 +77,7 @@ __tests__ = ( { "#url" : "https://civitai.com/images/26962948", "#class": civitai.CivitaiImageExtractor, - "#options" : {"quality": "w"}, + "#options" : {"quality": "w", "metadata": True}, "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png", "#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671", @@ -92,17 +92,52 @@ __tests__ = ( "postId" : 6030721, "stats" : dict, "url" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b", + "uuid" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b", + "width" : 1152, "user" : { "username": "bolero537", }, - "uuid" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b", - "width" : 1152, + "generation": { + "canRemix" : True, + "external" : None, + "generationProcess": "img2img", + "resources" : list, + "techniques": [], + "tools" : [], + "meta" : { + "Denoising strength": "0.4", + "Model" : "boleromix_XL_V1.3", + "Model hash" : "afaf521da2", + "Size" : "1152x1536", + "Tiled Diffusion scale factor": "1.5", + "Tiled Diffusion upscaler": "R-ESRGAN 4x+ Anime6B", + "VAE" : "sdxl_vae.safetensors", + "Version" : "v1.7.0", + "cfgScale" : 7, + "negativePrompt": "negativeXL_D,(worst quality,extra legs,extra arms,extra ears,bad fingers,extra fingers,bad anatomy, missing fingers, lowres,username, artist name, text,pubic hair,bar censor,censored,multipul angle,split view,realistic,3D:1)", + "prompt" : "masterpiece,ultra-detailed,best quality,8K,illustration,cute face,clean skin ,shiny hair,girl,ultra-detailed-eyes,simple background, maid, maid apron, maid headdress, long sleeves,tray,tea,cup,skirt lift", + "resources" : list, + "sampler" : "DPM++ 2M Karras", + "seed" : 3150861441, + "steps" : 20, + "hashes" : { + "lora:add-detail-xl": "9c783c8ce46c", + "lora:classic maid_XL_V1.0": "e8f6e4297112", + "model": "afaf521da2", + "vae": "735e4c3a44", + }, + "TI hashes" : { + "negativeXL_D": "fff5d51ab655", + }, + }, + }, }, { - "#url" : "https://civitai.com/posts/6877551", - "#class": civitai.CivitaiPostExtractor, - "#urls" : [ + "#url" : "https://civitai.com/posts/6877551", + "#class" : civitai.CivitaiPostExtractor, + "#options": {"metadata": "generation"}, + "#urls" : [ "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6220fa0f-9037-4b1d-bfbd-a740a06eeb7c/original=true/30748752.png", "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cd1edb7f-7b50-4da5-bf23-d38f24d8aef0/original=true/30748747.png", "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cfd5b231-accd-49bd-8bde-370880f63aa6/original=true/30748733.png", @@ -117,6 +152,15 @@ __tests__ = ( "uuid": {"6220fa0f-9037-4b1d-bfbd-a740a06eeb7c", "cd1edb7f-7b50-4da5-bf23-d38f24d8aef0", "cfd5b231-accd-49bd-8bde-370880f63aa6"}, + "generation": { + "resources" : list, + "techniques": [], + "tools" : [], + "meta" : { + "prompt" : str, + "negativePrompt": str, + }, + }, }, },