diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py index 16bc78ffc..e456475e5 100644 --- a/yt_dlp/extractor/extractors.py +++ b/yt_dlp/extractor/extractors.py @@ -867,6 +867,7 @@ from .newgrounds import ( NewgroundsIE, NewgroundsPlaylistIE, + NewgroundsUserIE, ) from .newstube import NewstubeIE from .nextmedia import ( diff --git a/yt_dlp/extractor/newgrounds.py b/yt_dlp/extractor/newgrounds.py index 41549a2f1..25b468b7d 100644 --- a/yt_dlp/extractor/newgrounds.py +++ b/yt_dlp/extractor/newgrounds.py @@ -1,5 +1,7 @@ +# coding: utf-8 from __future__ import unicode_literals +import functools import re from .common import InfoExtractor @@ -8,8 +10,9 @@ int_or_none, parse_count, parse_duration, - parse_filesize, unified_timestamp, + OnDemandPagedList, + try_get, ) @@ -88,10 +91,10 @@ def _real_extract(self, url): webpage = self._download_webpage(url, media_id) title = self._html_search_regex( - r'([^>]+)', webpage, 'title') + r'(.+?)', webpage, 'title') media_url_string = self._search_regex( - r'"url"\s*:\s*("[^"]+"),', webpage, 'media url', default=None, fatal=False) + r'"url"\s*:\s*("[^"]+"),', webpage, 'media url', default=None) if media_url_string: media_url = self._parse_json(media_url_string, media_id) @@ -128,20 +131,26 @@ def _real_extract(self, url): (r'
\s*Uploaded\s*
\s*
([^<]+
\s*
[^<]+)', r'
\s*Uploaded\s*
\s*
([^<]+)'), webpage, 'timestamp', default=None)) - duration = parse_duration(self._search_regex( - r'(?s)
\s*Song\s*
\s*
.+?
\s*
([^<]+)', webpage, + duration = parse_duration(self._html_search_regex( + r'"duration"\s*:\s*["\']?([\d]+)["\']?,', webpage, 'duration', default=None)) - view_count = parse_count(self._html_search_regex(r'(?s)
\s*Views\s*
\s*
([\d\.,]+)
', webpage, - 'view_count', fatal=False, default=None)) + view_count = parse_count(self._html_search_regex( + r'(?s)
\s*Views\s*
\s*
([\d\.,]+)
', webpage, + 'view count', default=None)) - filesize_approx = parse_filesize(self._html_search_regex( - r'(?s)
\s*Song\s*
\s*
(.+?)
', webpage, 'filesize', + filesize = int_or_none(self._html_search_regex( + r'"filesize"\s*:\s*["\']?([\d]+)["\']?,', webpage, 'filesize', default=None)) - if len(formats) == 1: - formats[0]['filesize_approx'] = filesize_approx - if '
Song' in webpage: + video_type_description = self._html_search_regex( + r'"description"\s*:\s*["\']?([^"\']+)["\']?,', webpage, 'filesize', + default=None) + + if len(formats) == 1: + formats[0]['filesize'] = filesize + + if video_type_description == 'Audio File': formats[0]['vcodec'] = 'none' self._check_formats(formats, media_id) self._sort_formats(formats) @@ -160,6 +169,7 @@ def _real_extract(self, url): class NewgroundsPlaylistIE(InfoExtractor): + IE_NAME = 'Newgrounds:playlist' _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:collection|[^/]+/search/[^/]+)/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://www.newgrounds.com/collection/cats', @@ -202,7 +212,57 @@ def _real_extract(self, url): continue entries.append( self.url_result( - 'https://www.newgrounds.com/%s' % path, + f'https://www.newgrounds.com/{path}', ie=NewgroundsIE.ie_key(), video_id=media_id)) return self.playlist_result(entries, playlist_id, title) + + +class NewgroundsUserIE(InfoExtractor): + IE_NAME = 'Newgrounds:user' + _VALID_URL = r'https?://(?P[^\.]+)\.newgrounds\.com/(?:movies|audio)/?(?:[#?]|$)' + _TESTS = [{ + 'url': 'https://burn7.newgrounds.com/audio', + 'info_dict': { + 'id': 'burn7', + }, + 'playlist_mincount': 150, + }, { + 'url': 'https://burn7.newgrounds.com/movies', + 'info_dict': { + 'id': 'burn7', + }, + 'playlist_mincount': 2, + }, { + 'url': 'https://brian-beaton.newgrounds.com/movies', + 'info_dict': { + 'id': 'brian-beaton', + }, + 'playlist_mincount': 10, + }] + _PAGE_SIZE = 30 + + def _fetch_page(self, channel_id, url, page): + page += 1 + posts_info = self._download_json( + f'{url}/page/{page}', channel_id, + note=f'Downloading page {page}', headers={ + 'Accept': 'application/json, text/javascript, */*; q = 0.01', + 'X-Requested-With': 'XMLHttpRequest', + }) + sequence = posts_info.get('sequence', []) + for year in sequence: + posts = try_get(posts_info, lambda x: x['years'][str(year)]['items']) + for post in posts: + path, media_id = self._search_regex( + r']+\bhref=["\'][^"\']+((?:portal/view|audio/listen)/(\d+))[^>]+>', + post, 'url', group=(1, 2)) + yield self.url_result(f'https://www.newgrounds.com/{path}', NewgroundsIE.ie_key(), media_id) + + def _real_extract(self, url): + channel_id = self._match_id(url) + + entries = OnDemandPagedList(functools.partial( + self._fetch_page, channel_id, url), self._PAGE_SIZE) + + return self.playlist_result(entries, channel_id)