1
0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-22 02:32:33 +01:00

merge #5041: [steamgriddb] add support (#5033)

This commit is contained in:
Mike Fährmann 2024-01-13 00:59:15 +01:00
commit 355b909f46
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
6 changed files with 516 additions and 0 deletions

View File

@ -3089,6 +3089,176 @@ Description
Download video files.
extractor.steamgriddb.animated
------------------------------
Type
``bool``
Default
``true``
Description
Include animated assets when downloading from a list of assets.
extractor.steamgriddb.epilepsy
------------------------------
Type
``bool``
Default
``true``
Description
Include assets tagged with epilepsy when downloading from a list of assets.
extractor.steamgriddb.dimensions
--------------------------------
Type
* ``string``
* ``list`` of ``strings``
Default
``"all"``
Examples
* ``"1024x512,512x512"``
* ``["460x215", "920x430"]``
Description
Only include assets that are in the specified dimensions. ``all`` can be
used to specify all dimensions. Valid values are:
* Grids: ``460x215``, ``920x430``, ``600x900``, ``342x482``, ``660x930``,
``512x512``, ``1024x1024``
* Heroes: ``1920x620``, ``3840x1240``, ``1600x650``
* Logos: N/A (will be ignored)
* Icons: ``8x8``, ``10x10``, ``14x14``, ``16x16``, ``20x20``, ``24x24``,
``28x28``, ``32x32``, ``35x35``, ``40x40``, ``48x48``, ``54x54``,
``56x56``, ``57x57``, ``60x60``, ``64x64``, ``72x72``, ``76x76``,
``80x80``, ``90x90``, ``96x96``, ``100x100``, ``114x114``, ``120x120``,
``128x128``, ``144x144``, ``150x150``, ``152x152``, ``160x160``,
``180x180``, ``192x192``, ``194x194``, ``256x256``, ``310x310``,
``512x512``, ``768x768``, ``1024x1024``
extractor.steamgriddb.file-types
--------------------------------
Type
* ``string``
* ``list`` of ``strings``
Default
``"all"``
Examples
* ``"png,jpeg"``
* ``["jpeg", "webp"]``
Description
Only include assets that are in the specified file types. ``all`` can be
used to specifiy all file types. Valid values are:
* Grids: ``png``, ``jpeg``, ``jpg``, ``webp``
* Heroes: ``png``, ``jpeg``, ``jpg``, ``webp``
* Logos: ``png``, ``webp``
* Icons: ``png``, ``ico``
extractor.steamgriddb.download-fake-png
---------------------------------------
Type
``bool``
Default
``true``
Description
Download fake PNGs alongside the real file.
extractor.steamgriddb.humor
---------------------------
Type
``bool``
Default
``true``
Description
Include assets tagged with humor when downloading from a list of assets.
extractor.steamgriddb.languages
-------------------------------
Type
* ``string``
* ``list`` of ``strings``
Default
``"all"``
Examples
* ``"en,km"``
* ``["fr", "it"]``
Description
Only include assets that are in the specified languages. ``all`` can be
used to specifiy all languages. Valid values are `ISO 639-1 <https://en.wikipedia.org/wiki/ISO_639-1>`__
language codes.
extractor.steamgriddb.nsfw
--------------------------
Type
``bool``
Default
``true``
Description
Include assets tagged with adult content when downloading from a list of assets.
extractor.steamgriddb.sort
--------------------------
Type
``string``
Default
``score_desc``
Description
Set the chosen sorting method when downloading from a list of assets. Can be one of:
* ``score_desc`` (Highest Score (Beta))
* ``score_asc`` (Lowest Score (Beta))
* ``score_old_desc`` (Highest Score (Old))
* ``score_old_asc`` (Lowest Score (Old))
* ``age_desc`` (Newest First)
* ``age_asc`` (Oldest First)
extractor.steamgriddb.static
----------------------------
Type
``bool``
Default
``true``
Description
Include static assets when downloading from a list of assets.
extractor.steamgriddb.styles
----------------------------
Type
* ``string``
* ``list`` of ``strings``
Default
``all``
Examples
* ``white,black``
* ``["no_logo", "white_logo"]``
Description
Only include assets that are in the specified styles. ``all`` can be used
to specify all styles. Valid values are:
* Grids: ``alternate``, ``blurred``, ``no_logo``, ``material``, ``white_logo``
* Heroes: ``alternate``, ``blurred``, ``material``
* Logos: ``official``, ``white``, ``black``, ``custom``
* Icons: ``official``, ``custom``
extractor.steamgriddb.untagged
------------------------------
Type
``bool``
Default
``true``
Description
Include untagged assets when downloading from a list of assets.
extractor.[szurubooru].username & .token
----------------------------------------
Type

View File

@ -817,6 +817,12 @@ Consider all listed sites to potentially be NSFW.
<td>Presentations</td>
<td></td>
</tr>
<tr>
<td>SteamGridDB</td>
<td>https://www.steamgriddb.com</td>
<td>Individual Assets, Grids, Heroes, Icons, Logos</td>
<td></td>
</tr>
<tr>
<td>SubscribeStar</td>
<td>https://www.subscribestar.com/</td>

View File

@ -146,6 +146,7 @@ modules = [
"smugmug",
"soundgasm",
"speakerdeck",
"steamgriddb",
"subscribestar",
"szurubooru",
"tapas",

View File

@ -0,0 +1,211 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://www.steamgriddb.com"""
from .common import Extractor, Message
from .. import text, exception
BASE_PATTERN = r"(?:https?://)?(?:www\.)?steamgriddb\.com"
LANGUAGE_CODES = (
"aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av", "ay", "az",
"ba", "be", "bg", "bh", "bi", "bm", "bn", "bo", "br", "bs", "ca", "ce",
"ch", "co", "cr", "cs", "cu", "cv", "cy", "da", "de", "dv", "dz", "ee",
"el", "en", "eo", "es", "et", "eu", "fa", "ff", "fi", "fj", "fo", "fr",
"fy", "ga", "gd", "gl", "gn", "gu", "gv", "ha", "he", "hi", "ho", "hr",
"ht", "hu", "hy", "hz", "ia", "id", "ie", "ig", "ii", "ik", "io", "is",
"it", "iu", "ja", "jv", "ka", "kg", "ki", "kj", "kk", "kl", "km", "kn",
"ko", "kr", "ks", "ku", "kv", "kw", "ky", "la", "lb", "lg", "li", "ln",
"lo", "lt", "lu", "lv", "mg", "mh", "mi", "mk", "ml", "mn", "mr", "ms",
"mt", "my", "na", "nb", "nd", "ne", "ng", "nl", "nn", "no", "nr", "nv",
"ny", "oc", "oj", "om", "or", "os", "pa", "pi", "pl", "ps", "pt", "qu",
"rm", "rn", "ro", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk",
"sl", "sm", "sn", "so", "sq", "sr", "ss", "st", "su", "sv", "sw", "ta",
"te", "tg", "th", "ti", "tk", "tl", "tn", "to", "tr", "ts", "tt", "tw",
"ty", "ug", "uk", "ur", "uz", "ve", "vi", "vo", "wa", "wo", "xh", "yi",
"yo", "za", "zh", "zu",
)
FILE_EXT_TO_MIME = {
"png": "image/png",
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"webp": "image/webp",
"ico": "image/vnd.microsoft.icon",
"all": "all",
}
class SteamgriddbExtractor(Extractor):
"""Base class for SteamGridDB"""
category = "steamgriddb"
directory_fmt = ("{category}", "{subcategory}", "{game[id]}")
filename_fmt = "{game[id]}_{id}_{num:>02}.{extension}"
archive_fmt = "{filename}"
root = "https://www.steamgriddb.com"
def _init(self):
self.cookies_update({
"userprefs": "%7B%22adult%22%3Afalse%7D",
})
def items(self):
download_fake_png = self.config("download-fake-png", True)
for asset in self.assets():
if download_fake_png and asset.get("fake_png"):
urls = (asset["url"], asset["fake_png"])
else:
urls = (asset["url"],)
asset["count"] = len(urls)
yield Message.Directory, asset
for asset["num"], url in enumerate(urls, 1):
yield Message.Url, url, text.nameext_from_url(url, asset)
def _call(self, endpoint, **kwargs):
data = self.request(self.root + endpoint, **kwargs).json()
if not data["success"]:
raise exception.StopExtraction(data["error"])
return data["data"]
class SteamgriddbAssetsExtractor(SteamgriddbExtractor):
"""Base class for extracting a list of assets"""
def __init__(self, match):
SteamgriddbExtractor.__init__(self, match)
list_type = match.group(1)
id = int(match.group(2))
self.game_id = id if list_type == "game" else None
self.collection_id = id if list_type == "collection" else None
self.page = int(match.group(3) or 1)
def assets(self):
limit = 48
page = min(self.page - 1, 0)
sort = self.config("sort", "score_desc")
if sort not in ("score_desc", "score_asc", "score_old_desc",
"score_old_asc", "age_desc", "age_asc"):
raise exception.StopExtractor("Invalid sort '%s'", sort)
json = {
"static" : self.config("static", True),
"animated": self.config("animated", True),
"humor" : self.config("humor", True),
"nsfw" : self.config("nsfw", True),
"epilepsy": self.config("epilepsy", True),
"untagged": self.config("untagged", True),
"asset_type": self.asset_type,
"limit": limit,
"order": sort,
}
if self.valid_dimensions:
json["dimensions"] = self.config_list(
"dimensions", "dimension", self.valid_dimensions)
json["styles"] = self.config_list("styles", "style", self.valid_styles)
json["languages"] = self.config_list(
"languages", "language", LANGUAGE_CODES)
file_types = self.config_list(
"file-types", "file type", self.valid_file_types)
json["mime"] = [FILE_EXT_TO_MIME[i] for i in file_types]
if self.game_id:
json["game_id"] = [self.game_id]
else:
json["collection_id"] = self.collection_id
while True:
json["page"] = page
data = self._call(
"/api/public/search/assets", method="POST", json=json)
for asset in data["assets"]:
if not asset.get("game"):
asset["game"] = data["game"]
yield asset
if data["total"] <= limit * page:
break
page += 1
def config_list(self, key, type_name, valid_values):
value = self.config(key)
if isinstance(value, str):
value = value.split(",")
if value is None or "all" in value:
return ["all"]
for i in value:
if i not in valid_values:
raise exception.StopExtraction("Invalid %s '%s'", type_name, i)
return value
class SteamgriddbAssetExtractor(SteamgriddbExtractor):
"""Extractor for a single asset"""
subcategory = "asset"
pattern = BASE_PATTERN + r"/(grid|hero|logo|icon)/(\d+)"
example = "https://www.steamgriddb.com/grid/1234"
def __init__(self, match):
SteamgriddbExtractor.__init__(self, match)
self.asset_type = match.group(1)
self.asset_id = match.group(2)
def assets(self):
endpoint = "/api/public/asset/" + self.asset_type + "/" + self.asset_id
asset = self._call(endpoint)["asset"]
return (asset,)
class SteamgriddbGridsExtractor(SteamgriddbAssetsExtractor):
subcategory = "grids"
asset_type = "grid"
pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/grids(?:/(\d+))?"
example = "https://www.steamgriddb.com/game/1234/grids"
valid_dimensions = ("460x215", "920x430", "600x900", "342x482", "660x930",
"512x512", "1024x1024")
valid_styles = ("alternate", "blurred", "no_logo", "material",
"white_logo")
valid_file_types = ("png", "jpeg", "jpg", "webp")
class SteamgriddbHeroesExtractor(SteamgriddbAssetsExtractor):
subcategory = "heroes"
asset_type = "hero"
pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/heroes(?:/(\d+))?"
example = "https://www.steamgriddb.com/game/1234/heroes"
valid_dimensions = ("1920x620", "3840x1240", "1600x650")
valid_styles = ("alternate", "blurred", "material")
valid_file_types = ("png", "jpeg", "jpg", "webp")
class SteamgriddbLogosExtractor(SteamgriddbAssetsExtractor):
subcategory = "logos"
asset_type = "logo"
pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/logos(?:/(\d+))?"
example = "https://www.steamgriddb.com/game/1234/logos"
valid_dimensions = None
valid_styles = ("official", "white", "black", "custom")
valid_file_types = ("png", "webp")
class SteamgriddbIconsExtractor(SteamgriddbAssetsExtractor):
subcategory = "icons"
asset_type = "icon"
pattern = BASE_PATTERN + r"/(game|collection)/(\d+)/icons(?:/(\d+))?"
example = "https://www.steamgriddb.com/game/1234/icons"
valid_dimensions = ["{0}x{0}".format(i) for i in (8, 10, 14, 16, 20, 24,
28, 32, 35, 40, 48, 54, 56, 57, 60, 64, 72, 76, 80, 90,
96, 100, 114, 120, 128, 144, 150, 152, 160, 180, 192,
194, 256, 310, 512, 768, 1024)]
valid_styles = ("official", "custom")
valid_file_types = ("png", "ico")

View File

@ -121,6 +121,7 @@ CATEGORY_MAP = {
"slideshare" : "SlideShare",
"smugmug" : "SmugMug",
"speakerdeck" : "Speaker Deck",
"steamgriddb" : "SteamGridDB",
"subscribestar" : "SubscribeStar",
"tbib" : "The Big ImageBoard",
"tcbscans" : "TCB Scans",
@ -267,6 +268,9 @@ SUBCATEGORY_MAP = {
"smugmug": {
"path": "Images from Users and Folders",
},
"steamgriddb": {
"asset": "Individual Assets",
},
"tumblr": {
"day": "Days",
},

124
test/results/steamgriddb.py Normal file
View File

@ -0,0 +1,124 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
from gallery_dl.extractor import steamgriddb
__tests__ = (
{
"#url" : "https://www.steamgriddb.com/grid/368023",
"#category": ("", "steamgriddb", "asset"),
"#class" : steamgriddb.SteamgriddbAssetExtractor,
"#urls" : ("https://cdn2.steamgriddb.com/grid/"
"82fee171d62c044898d99ba0fddeb203.png"),
"#count" : 1,
"#sha1_content": "0bffaccae6f35f9fab529684a5b158d1cec4186b",
"game": {
"id" : 5259324,
"name": "Helltaker",
},
},
{
"#url" : "https://www.steamgriddb.com/grid/132605",
"#category": ("", "steamgriddb", "asset"),
"#class" : steamgriddb.SteamgriddbAssetExtractor,
"#count" : 2,
"#sha1_url" : "4ff9158c008a1f01921d7553bcabf5e6204cdc79",
"#sha1_content": "bc16c5eebf71463abdb33cfbf4b45a2fe092a2b2",
"game": {
"id" : 5247997,
"name": "OMORI",
},
},
{
"#url" : "https://www.steamgriddb.com/grid/132605",
"#category": ("", "steamgriddb", "asset"),
"#class" : steamgriddb.SteamgriddbAssetExtractor,
"#options" : {"download-fake-png": False},
"#count" : 1,
"#sha1_url" : "f6819c593ff65f15864796fb89581f05d21adddb",
"#sha1_content": "0d9e6114dd8bb9699182fbb7c6bd9064d8b0b6cd",
"game": {
"id" : 5247997,
"name": "OMORI",
},
},
{
"#url" : "https://www.steamgriddb.com/hero/61104",
"#category": ("", "steamgriddb", "asset"),
"#class" : steamgriddb.SteamgriddbAssetExtractor,
},
{
"#url" : "https://www.steamgriddb.com/logo/9610",
"#category": ("", "steamgriddb", "asset"),
"#class" : steamgriddb.SteamgriddbAssetExtractor,
},
{
"#url" : "https://www.steamgriddb.com/icon/173",
"#category": ("", "steamgriddb", "asset"),
"#class" : steamgriddb.SteamgriddbAssetExtractor,
},
{
"#url" : "https://www.steamgriddb.com/game/5259324/grids",
"#category": ("", "steamgriddb", "grids"),
"#class" : steamgriddb.SteamgriddbGridsExtractor,
"#range" : "1-10",
"#count" : 10,
},
{
"#url" : "https://www.steamgriddb.com/game/5259324/grids",
"#category": ("", "steamgriddb", "grids"),
"#class" : steamgriddb.SteamgriddbGridsExtractor,
"#options" : {"humor": False, "epilepsy": False, "untagged": False},
"#range" : "1-33",
"#count" : 33,
},
{
"#url" : "https://www.steamgriddb.com/game/5331605/heroes",
"#category": ("", "steamgriddb", "heroes"),
"#class" : steamgriddb.SteamgriddbHeroesExtractor,
},
{
"#url" : "https://www.steamgriddb.com/game/5255394/logos",
"#category": ("", "steamgriddb", "logos"),
"#class" : steamgriddb.SteamgriddbLogosExtractor,
},
{
"#url" : "https://www.steamgriddb.com/game/5279790/icons",
"#category": ("", "steamgriddb", "icons"),
"#class" : steamgriddb.SteamgriddbIconsExtractor,
},
{
"#url" : "https://www.steamgriddb.com/collection/332/grids",
"#category": ("", "steamgriddb", "grids"),
"#class" : steamgriddb.SteamgriddbGridsExtractor,
"#range" : "1-10",
"#count" : 10,
},
{
"#url" : "https://www.steamgriddb.com/collection/332/heroes",
"#category": ("", "steamgriddb", "heroes"),
"#class" : steamgriddb.SteamgriddbHeroesExtractor,
"#options" : {"animated": False},
"#count" : 0,
},
)