diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 312cdc23..1bbfa12e 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -790,7 +790,7 @@ Consider all listed sites to potentially be NSFW. Skeb https://skeb.jp/ - Followed Users, Posts, Search Results, User Profiles + Followed Creators, Followed Users, Posts, Search Results, User Profiles diff --git a/gallery_dl/extractor/skeb.py b/gallery_dl/extractor/skeb.py index b9b4b3c4..38a2d166 100644 --- a/gallery_dl/extractor/skeb.py +++ b/gallery_dl/extractor/skeb.py @@ -26,10 +26,10 @@ class SkebExtractor(Extractor): def _init(self): self.thumbnails = self.config("thumbnails", False) self.article = self.config("article", False) - self.headers = { - "Accept" : "application/json, text/plain, */*", - "Authorization": "Bearer null", - } + self.headers = {"Accept": "application/json, text/plain, */*"} + + if "Authorization" not in self.session.headers: + self.headers["Authorization"] = "Bearer null" def request(self, url, **kwargs): while True: @@ -55,6 +55,12 @@ class SkebExtractor(Extractor): url = file["file_url"] yield Message.Url, url, text.nameext_from_url(url, post) + def _items_users(self): + base = self.root + "/@" + for user in self.users(): + user["_extractor"] = SkebUserExtractor + yield Message.Queue, base + user["screen_name"], user + def posts(self): """Return post number""" @@ -83,6 +89,20 @@ class SkebExtractor(Extractor): return params["offset"] += 30 + def _pagination_users(self, endpoint, params): + url = "{}/api{}".format(self.root, endpoint) + params["offset"] = 0 + params["limit"] = 90 + + while True: + data = self.request( + url, params=params, headers=self.headers).json() + yield from data + + if len(data) < params["limit"]: + return + params["offset"] += params["limit"] + def _get_post_data(self, user_name, post_num): url = "{}/api/users/{}/works/{}".format( self.root, user_name, post_num) @@ -256,22 +276,23 @@ class SkebFollowingExtractor(SkebExtractor): pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/following_creators" example = "https://skeb.jp/@USER/following_creators" - def items(self): - for user in self.users(): - url = "{}/@{}".format(self.root, user["screen_name"]) - user["_extractor"] = SkebUserExtractor - yield Message.Queue, url, user + items = SkebExtractor._items_users def users(self): - url = "{}/api/users/{}/following_creators".format( - self.root, self.user_name) - params = {"sort": "date", "offset": 0, "limit": 90} + endpoint = "/users/{}/following_creators".format(self.user_name) + params = {"sort": "date"} + return self._pagination_users(endpoint, params) - while True: - data = self.request( - url, params=params, headers=self.headers).json() - yield from data - if len(data) < params["limit"]: - return - params["offset"] += params["limit"] +class SkebFollowingUsersExtractor(SkebExtractor): + """Extractor for your followed users""" + subcategory = "following-users" + pattern = r"(?:https?://)?skeb\.jp/following_users()" + example = "https://skeb.jp/following_users" + + items = SkebExtractor._items_users + + def users(self): + endpoint = "/following_users" + params = {} + return self._pagination_users(endpoint, params) diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index ecdc121d..47d7ef92 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -273,6 +273,10 @@ SUBCATEGORY_MAP = { "sexcom": { "pins": "User Pins", }, + "skeb": { + "following" : "Followed Creators", + "following-users": "Followed Users", + }, "smugmug": { "path": "Images from Users and Folders", }, diff --git a/test/results/skeb.py b/test/results/skeb.py index a05ed63e..4aa8691d 100644 --- a/test/results/skeb.py +++ b/test/results/skeb.py @@ -82,4 +82,12 @@ __tests__ = ( "#class" : skeb.SkebFollowingExtractor, }, +{ + "#url" : "https://skeb.jp/following_users", + "#category": ("", "skeb", "following-users"), + "#class" : skeb.SkebFollowingUsersExtractor, + "#pattern" : skeb.SkebUserExtractor.pattern, + "#auth" : True, +}, + )