mirror of
https://github.com/blackjack4494/yt-dlc.git
synced 2024-08-19 00:39:42 +02:00
Merge branch 'ytdl-org-master'
This commit is contained in:
commit
11f96ac427
21
ChangeLog
21
ChangeLog
@ -1,3 +1,24 @@
|
|||||||
|
version 2020.09.20
|
||||||
|
|
||||||
|
Core
|
||||||
|
* [extractor/common] Relax interaction count extraction in _json_ld
|
||||||
|
+ [extractor/common] Extract author as uploader for VideoObject in _json_ld
|
||||||
|
* [downloader/hls] Fix incorrect end byte in Range HTTP header for
|
||||||
|
media segments with EXT-X-BYTERANGE (#14748, #24512)
|
||||||
|
* [extractor/common] Handle ssl.CertificateError in _request_webpage (#26601)
|
||||||
|
* [downloader/http] Improve timeout detection when reading block of data
|
||||||
|
(#10935)
|
||||||
|
* [downloader/http] Retry download when urlopen times out (#10935, #26603)
|
||||||
|
|
||||||
|
Extractors
|
||||||
|
* [redtube] Extend URL regular expression (#26506)
|
||||||
|
* [twitch] Refactor
|
||||||
|
* [twitch:stream] Switch to GraphQL and fix reruns (#26535)
|
||||||
|
+ [telequebec] Add support for brightcove videos (#25833)
|
||||||
|
* [pornhub] Extract metadata from JSON-LD (#26614)
|
||||||
|
* [pornhub] Fix view count extraction (#26621, #26614)
|
||||||
|
|
||||||
|
|
||||||
version 2020.09.14
|
version 2020.09.14
|
||||||
|
|
||||||
Core
|
Core
|
||||||
|
@ -141,7 +141,7 @@ class HlsFD(FragmentFD):
|
|||||||
count = 0
|
count = 0
|
||||||
headers = info_dict.get('http_headers', {})
|
headers = info_dict.get('http_headers', {})
|
||||||
if byte_range:
|
if byte_range:
|
||||||
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'])
|
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
|
||||||
while count <= fragment_retries:
|
while count <= fragment_retries:
|
||||||
try:
|
try:
|
||||||
success, frag_content = self._download_fragment(
|
success, frag_content = self._download_fragment(
|
||||||
|
@ -106,7 +106,12 @@ class HttpFD(FileDownloader):
|
|||||||
set_range(request, range_start, range_end)
|
set_range(request, range_start, range_end)
|
||||||
# Establish connection
|
# Establish connection
|
||||||
try:
|
try:
|
||||||
ctx.data = self.ydl.urlopen(request)
|
try:
|
||||||
|
ctx.data = self.ydl.urlopen(request)
|
||||||
|
except (compat_urllib_error.URLError, ) as err:
|
||||||
|
if isinstance(err.reason, socket.timeout):
|
||||||
|
raise RetryDownload(err)
|
||||||
|
raise err
|
||||||
# When trying to resume, Content-Range HTTP header of response has to be checked
|
# When trying to resume, Content-Range HTTP header of response has to be checked
|
||||||
# to match the value of requested Range HTTP header. This is due to a webservers
|
# to match the value of requested Range HTTP header. This is due to a webservers
|
||||||
# that don't support resuming and serve a whole file with no Content-Range
|
# that don't support resuming and serve a whole file with no Content-Range
|
||||||
@ -218,9 +223,10 @@ class HttpFD(FileDownloader):
|
|||||||
|
|
||||||
def retry(e):
|
def retry(e):
|
||||||
to_stdout = ctx.tmpfilename == '-'
|
to_stdout = ctx.tmpfilename == '-'
|
||||||
if not to_stdout:
|
if ctx.stream is not None:
|
||||||
ctx.stream.close()
|
if not to_stdout:
|
||||||
ctx.stream = None
|
ctx.stream.close()
|
||||||
|
ctx.stream = None
|
||||||
ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
|
ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
|
||||||
raise RetryDownload(e)
|
raise RetryDownload(e)
|
||||||
|
|
||||||
@ -233,9 +239,11 @@ class HttpFD(FileDownloader):
|
|||||||
except socket.timeout as e:
|
except socket.timeout as e:
|
||||||
retry(e)
|
retry(e)
|
||||||
except socket.error as e:
|
except socket.error as e:
|
||||||
if e.errno not in (errno.ECONNRESET, errno.ETIMEDOUT):
|
# SSLError on python 2 (inherits socket.error) may have
|
||||||
raise
|
# no errno set but this error message
|
||||||
retry(e)
|
if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or getattr(e, 'message', None) == 'The read operation timed out':
|
||||||
|
retry(e)
|
||||||
|
raise
|
||||||
|
|
||||||
byte_counter += len(data_block)
|
byte_counter += len(data_block)
|
||||||
|
|
||||||
|
@ -10,6 +10,7 @@ import os
|
|||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
import socket
|
import socket
|
||||||
|
import ssl
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import math
|
import math
|
||||||
@ -67,6 +68,7 @@ from ..utils import (
|
|||||||
sanitized_Request,
|
sanitized_Request,
|
||||||
sanitize_filename,
|
sanitize_filename,
|
||||||
str_or_none,
|
str_or_none,
|
||||||
|
str_to_int,
|
||||||
strip_or_none,
|
strip_or_none,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
@ -623,9 +625,12 @@ class InfoExtractor(object):
|
|||||||
url_or_request = update_url_query(url_or_request, query)
|
url_or_request = update_url_query(url_or_request, query)
|
||||||
if data is not None or headers:
|
if data is not None or headers:
|
||||||
url_or_request = sanitized_Request(url_or_request, data, headers)
|
url_or_request = sanitized_Request(url_or_request, data, headers)
|
||||||
|
exceptions = [compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error]
|
||||||
|
if hasattr(ssl, 'CertificateError'):
|
||||||
|
exceptions.append(ssl.CertificateError)
|
||||||
try:
|
try:
|
||||||
return self._downloader.urlopen(url_or_request)
|
return self._downloader.urlopen(url_or_request)
|
||||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
except tuple(exceptions) as err:
|
||||||
if isinstance(err, compat_urllib_error.HTTPError):
|
if isinstance(err, compat_urllib_error.HTTPError):
|
||||||
if self.__can_accept_status_code(err, expected_status):
|
if self.__can_accept_status_code(err, expected_status):
|
||||||
# Retain reference to error to prevent file object from
|
# Retain reference to error to prevent file object from
|
||||||
@ -1244,7 +1249,10 @@ class InfoExtractor(object):
|
|||||||
interaction_type = is_e.get('interactionType')
|
interaction_type = is_e.get('interactionType')
|
||||||
if not isinstance(interaction_type, compat_str):
|
if not isinstance(interaction_type, compat_str):
|
||||||
continue
|
continue
|
||||||
interaction_count = int_or_none(is_e.get('userInteractionCount'))
|
# For interaction count some sites provide string instead of
|
||||||
|
# an integer (as per spec) with non digit characters (e.g. ",")
|
||||||
|
# so extracting count with more relaxed str_to_int
|
||||||
|
interaction_count = str_to_int(is_e.get('userInteractionCount'))
|
||||||
if interaction_count is None:
|
if interaction_count is None:
|
||||||
continue
|
continue
|
||||||
count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1])
|
count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1])
|
||||||
@ -1264,6 +1272,7 @@ class InfoExtractor(object):
|
|||||||
'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
|
'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
|
||||||
'duration': parse_duration(e.get('duration')),
|
'duration': parse_duration(e.get('duration')),
|
||||||
'timestamp': unified_timestamp(e.get('uploadDate')),
|
'timestamp': unified_timestamp(e.get('uploadDate')),
|
||||||
|
'uploader': str_or_none(e.get('author')),
|
||||||
'filesize': float_or_none(e.get('contentSize')),
|
'filesize': float_or_none(e.get('contentSize')),
|
||||||
'tbr': int_or_none(e.get('bitrate')),
|
'tbr': int_or_none(e.get('bitrate')),
|
||||||
'width': int_or_none(e.get('width')),
|
'width': int_or_none(e.get('width')),
|
||||||
|
@ -17,6 +17,7 @@ from ..utils import (
|
|||||||
determine_ext,
|
determine_ext,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
merge_dicts,
|
||||||
NO_DEFAULT,
|
NO_DEFAULT,
|
||||||
orderedSet,
|
orderedSet,
|
||||||
remove_quotes,
|
remove_quotes,
|
||||||
@ -59,13 +60,14 @@ class PornHubIE(PornHubBaseIE):
|
|||||||
'''
|
'''
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
|
||||||
'md5': '1e19b41231a02eba417839222ac9d58e',
|
'md5': 'a6391306d050e4547f62b3f485dd9ba9',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '648719015',
|
'id': '648719015',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
|
'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
|
||||||
'uploader': 'Babes',
|
'uploader': 'Babes',
|
||||||
'upload_date': '20130628',
|
'upload_date': '20130628',
|
||||||
|
'timestamp': 1372447216,
|
||||||
'duration': 361,
|
'duration': 361,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
'like_count': int,
|
'like_count': int,
|
||||||
@ -82,8 +84,8 @@ class PornHubIE(PornHubBaseIE):
|
|||||||
'id': '1331683002',
|
'id': '1331683002',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': '重庆婷婷女王足交',
|
'title': '重庆婷婷女王足交',
|
||||||
'uploader': 'Unknown',
|
|
||||||
'upload_date': '20150213',
|
'upload_date': '20150213',
|
||||||
|
'timestamp': 1423804862,
|
||||||
'duration': 1753,
|
'duration': 1753,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
'like_count': int,
|
'like_count': int,
|
||||||
@ -121,6 +123,7 @@ class PornHubIE(PornHubBaseIE):
|
|||||||
'params': {
|
'params': {
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
'skip': 'This video has been disabled',
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
@ -338,10 +341,10 @@ class PornHubIE(PornHubBaseIE):
|
|||||||
|
|
||||||
video_uploader = self._html_search_regex(
|
video_uploader = self._html_search_regex(
|
||||||
r'(?s)From: .+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
|
r'(?s)From: .+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
|
||||||
webpage, 'uploader', fatal=False)
|
webpage, 'uploader', default=None)
|
||||||
|
|
||||||
view_count = self._extract_count(
|
view_count = self._extract_count(
|
||||||
r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
|
r'<span class="count">([\d,\.]+)</span> [Vv]iews', webpage, 'view')
|
||||||
like_count = self._extract_count(
|
like_count = self._extract_count(
|
||||||
r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
|
r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
|
||||||
dislike_count = self._extract_count(
|
dislike_count = self._extract_count(
|
||||||
@ -356,7 +359,11 @@ class PornHubIE(PornHubBaseIE):
|
|||||||
if div:
|
if div:
|
||||||
return re.findall(r'<a[^>]+\bhref=[^>]+>([^<]+)', div)
|
return re.findall(r'<a[^>]+\bhref=[^>]+>([^<]+)', div)
|
||||||
|
|
||||||
return {
|
info = self._search_json_ld(webpage, video_id, default={})
|
||||||
|
# description provided in JSON-LD is irrelevant
|
||||||
|
info['description'] = None
|
||||||
|
|
||||||
|
return merge_dicts({
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'uploader': video_uploader,
|
'uploader': video_uploader,
|
||||||
'upload_date': upload_date,
|
'upload_date': upload_date,
|
||||||
@ -372,7 +379,7 @@ class PornHubIE(PornHubBaseIE):
|
|||||||
'tags': extract_list('tags'),
|
'tags': extract_list('tags'),
|
||||||
'categories': extract_list('categories'),
|
'categories': extract_list('categories'),
|
||||||
'subtitles': subtitles,
|
'subtitles': subtitles,
|
||||||
}
|
}, info)
|
||||||
|
|
||||||
|
|
||||||
class PornHubPlaylistBaseIE(PornHubBaseIE):
|
class PornHubPlaylistBaseIE(PornHubBaseIE):
|
||||||
|
@ -15,7 +15,7 @@ from ..utils import (
|
|||||||
|
|
||||||
|
|
||||||
class RedTubeIE(InfoExtractor):
|
class RedTubeIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:(?:www\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P<id>[0-9]+)'
|
_VALID_URL = r'https?://(?:(?:\w+\.)?redtube\.com/|embed\.redtube\.com/\?.*?\bid=)(?P<id>[0-9]+)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.redtube.com/66418',
|
'url': 'http://www.redtube.com/66418',
|
||||||
'md5': 'fc08071233725f26b8f014dba9590005',
|
'md5': 'fc08071233725f26b8f014dba9590005',
|
||||||
@ -31,6 +31,9 @@ class RedTubeIE(InfoExtractor):
|
|||||||
}, {
|
}, {
|
||||||
'url': 'http://embed.redtube.com/?bgcolor=000000&id=1443286',
|
'url': 'http://embed.redtube.com/?bgcolor=000000&id=1443286',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://it.redtube.com/66418',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -13,14 +13,24 @@ from ..utils import (
|
|||||||
|
|
||||||
class TeleQuebecBaseIE(InfoExtractor):
|
class TeleQuebecBaseIE(InfoExtractor):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _limelight_result(media_id):
|
def _result(url, ie_key):
|
||||||
return {
|
return {
|
||||||
'_type': 'url_transparent',
|
'_type': 'url_transparent',
|
||||||
'url': smuggle_url(
|
'url': smuggle_url(url, {'geo_countries': ['CA']}),
|
||||||
'limelight:media:' + media_id, {'geo_countries': ['CA']}),
|
'ie_key': ie_key,
|
||||||
'ie_key': 'LimelightMedia',
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _limelight_result(media_id):
|
||||||
|
return TeleQuebecBaseIE._result(
|
||||||
|
'limelight:media:' + media_id, 'LimelightMedia')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _brightcove_result(brightcove_id):
|
||||||
|
return TeleQuebecBaseIE._result(
|
||||||
|
'http://players.brightcove.net/6150020952001/default_default/index.html?videoId=%s'
|
||||||
|
% brightcove_id, 'BrightcoveNew')
|
||||||
|
|
||||||
|
|
||||||
class TeleQuebecIE(TeleQuebecBaseIE):
|
class TeleQuebecIE(TeleQuebecBaseIE):
|
||||||
_VALID_URL = r'''(?x)
|
_VALID_URL = r'''(?x)
|
||||||
@ -37,11 +47,27 @@ class TeleQuebecIE(TeleQuebecBaseIE):
|
|||||||
'id': '577116881b4b439084e6b1cf4ef8b1b3',
|
'id': '577116881b4b439084e6b1cf4ef8b1b3',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Un petit choc et puis repart!',
|
'title': 'Un petit choc et puis repart!',
|
||||||
'description': 'md5:b04a7e6b3f74e32d7b294cffe8658374',
|
'description': 'md5:067bc84bd6afecad85e69d1000730907',
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://zonevideo.telequebec.tv/media/55267/le-soleil/passe-partout',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '6167180337001',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Le soleil',
|
||||||
|
'description': 'md5:64289c922a8de2abbe99c354daffde02',
|
||||||
|
'uploader_id': '6150020952001',
|
||||||
|
'upload_date': '20200625',
|
||||||
|
'timestamp': 1593090307,
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'format': 'bestvideo',
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
'add_ie': ['BrightcoveNew'],
|
||||||
}, {
|
}, {
|
||||||
# no description
|
# no description
|
||||||
'url': 'http://zonevideo.telequebec.tv/media/30261',
|
'url': 'http://zonevideo.telequebec.tv/media/30261',
|
||||||
@ -58,7 +84,14 @@ class TeleQuebecIE(TeleQuebecBaseIE):
|
|||||||
'https://mnmedias.api.telequebec.tv/api/v2/media/' + media_id,
|
'https://mnmedias.api.telequebec.tv/api/v2/media/' + media_id,
|
||||||
media_id)['media']
|
media_id)['media']
|
||||||
|
|
||||||
info = self._limelight_result(media_data['streamInfo']['sourceId'])
|
source_id = media_data['streamInfo']['sourceId']
|
||||||
|
source = (try_get(
|
||||||
|
media_data, lambda x: x['streamInfo']['source'],
|
||||||
|
compat_str) or 'limelight').lower()
|
||||||
|
if source == 'brightcove':
|
||||||
|
info = self._brightcove_result(source_id)
|
||||||
|
else:
|
||||||
|
info = self._limelight_result(source_id)
|
||||||
info.update({
|
info.update({
|
||||||
'title': media_data.get('title'),
|
'title': media_data.get('title'),
|
||||||
'description': try_get(
|
'description': try_get(
|
||||||
|
@ -24,7 +24,6 @@ from ..utils import (
|
|||||||
parse_duration,
|
parse_duration,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
qualities,
|
qualities,
|
||||||
str_or_none,
|
|
||||||
try_get,
|
try_get,
|
||||||
unified_timestamp,
|
unified_timestamp,
|
||||||
update_url_query,
|
update_url_query,
|
||||||
@ -337,19 +336,27 @@ def _make_video_result(node):
|
|||||||
class TwitchGraphQLBaseIE(TwitchBaseIE):
|
class TwitchGraphQLBaseIE(TwitchBaseIE):
|
||||||
_PAGE_LIMIT = 100
|
_PAGE_LIMIT = 100
|
||||||
|
|
||||||
def _download_gql(self, video_id, op, variables, sha256_hash, note, fatal=True):
|
_OPERATION_HASHES = {
|
||||||
|
'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
|
||||||
|
'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
|
||||||
|
'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
|
||||||
|
'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
|
||||||
|
'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
|
||||||
|
'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
|
||||||
|
'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
|
||||||
|
}
|
||||||
|
|
||||||
|
def _download_gql(self, video_id, ops, note, fatal=True):
|
||||||
|
for op in ops:
|
||||||
|
op['extensions'] = {
|
||||||
|
'persistedQuery': {
|
||||||
|
'version': 1,
|
||||||
|
'sha256Hash': self._OPERATION_HASHES[op['operationName']],
|
||||||
|
}
|
||||||
|
}
|
||||||
return self._download_json(
|
return self._download_json(
|
||||||
'https://gql.twitch.tv/gql', video_id, note,
|
'https://gql.twitch.tv/gql', video_id, note,
|
||||||
data=json.dumps({
|
data=json.dumps(ops).encode(),
|
||||||
'operationName': op,
|
|
||||||
'variables': variables,
|
|
||||||
'extensions': {
|
|
||||||
'persistedQuery': {
|
|
||||||
'version': 1,
|
|
||||||
'sha256Hash': sha256_hash,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}).encode(),
|
|
||||||
headers={
|
headers={
|
||||||
'Content-Type': 'text/plain;charset=UTF-8',
|
'Content-Type': 'text/plain;charset=UTF-8',
|
||||||
'Client-ID': self._CLIENT_ID,
|
'Client-ID': self._CLIENT_ID,
|
||||||
@ -369,14 +376,15 @@ class TwitchCollectionIE(TwitchGraphQLBaseIE):
|
|||||||
}]
|
}]
|
||||||
|
|
||||||
_OPERATION_NAME = 'CollectionSideBar'
|
_OPERATION_NAME = 'CollectionSideBar'
|
||||||
_SHA256_HASH = '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14'
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
collection_id = self._match_id(url)
|
collection_id = self._match_id(url)
|
||||||
collection = self._download_gql(
|
collection = self._download_gql(
|
||||||
collection_id, self._OPERATION_NAME,
|
collection_id, [{
|
||||||
{'collectionID': collection_id}, self._SHA256_HASH,
|
'operationName': self._OPERATION_NAME,
|
||||||
'Downloading collection GraphQL')['data']['collection']
|
'variables': {'collectionID': collection_id},
|
||||||
|
}],
|
||||||
|
'Downloading collection GraphQL')[0]['data']['collection']
|
||||||
title = collection.get('title')
|
title = collection.get('title')
|
||||||
entries = []
|
entries = []
|
||||||
for edge in collection['items']['edges']:
|
for edge in collection['items']['edges']:
|
||||||
@ -403,14 +411,16 @@ class TwitchPlaylistBaseIE(TwitchGraphQLBaseIE):
|
|||||||
if cursor:
|
if cursor:
|
||||||
variables['cursor'] = cursor
|
variables['cursor'] = cursor
|
||||||
page = self._download_gql(
|
page = self._download_gql(
|
||||||
channel_name, self._OPERATION_NAME, variables,
|
channel_name, [{
|
||||||
self._SHA256_HASH,
|
'operationName': self._OPERATION_NAME,
|
||||||
|
'variables': variables,
|
||||||
|
}],
|
||||||
'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num),
|
'Downloading %ss GraphQL page %s' % (self._NODE_KIND, page_num),
|
||||||
fatal=False)
|
fatal=False)
|
||||||
if not page:
|
if not page:
|
||||||
break
|
break
|
||||||
edges = try_get(
|
edges = try_get(
|
||||||
page, lambda x: x['data']['user'][entries_key]['edges'], list)
|
page, lambda x: x[0]['data']['user'][entries_key]['edges'], list)
|
||||||
if not edges:
|
if not edges:
|
||||||
break
|
break
|
||||||
for edge in edges:
|
for edge in edges:
|
||||||
@ -553,7 +563,6 @@ class TwitchVideosIE(TwitchPlaylistBaseIE):
|
|||||||
'views': 'Popular',
|
'views': 'Popular',
|
||||||
}
|
}
|
||||||
|
|
||||||
_SHA256_HASH = 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb'
|
|
||||||
_OPERATION_NAME = 'FilterableVideoTower_Videos'
|
_OPERATION_NAME = 'FilterableVideoTower_Videos'
|
||||||
_ENTRY_KIND = 'video'
|
_ENTRY_KIND = 'video'
|
||||||
_EDGE_KIND = 'VideoEdge'
|
_EDGE_KIND = 'VideoEdge'
|
||||||
@ -622,7 +631,6 @@ class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
|
|||||||
# NB: values other than 20 result in skipped videos
|
# NB: values other than 20 result in skipped videos
|
||||||
_PAGE_LIMIT = 20
|
_PAGE_LIMIT = 20
|
||||||
|
|
||||||
_SHA256_HASH = 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777'
|
|
||||||
_OPERATION_NAME = 'ClipsCards__User'
|
_OPERATION_NAME = 'ClipsCards__User'
|
||||||
_ENTRY_KIND = 'clip'
|
_ENTRY_KIND = 'clip'
|
||||||
_EDGE_KIND = 'ClipEdge'
|
_EDGE_KIND = 'ClipEdge'
|
||||||
@ -680,7 +688,6 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
|
|||||||
'playlist_mincount': 3,
|
'playlist_mincount': 3,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
_SHA256_HASH = '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84'
|
|
||||||
_OPERATION_NAME = 'ChannelCollectionsContent'
|
_OPERATION_NAME = 'ChannelCollectionsContent'
|
||||||
_ENTRY_KIND = 'collection'
|
_ENTRY_KIND = 'collection'
|
||||||
_EDGE_KIND = 'CollectionsItemEdge'
|
_EDGE_KIND = 'CollectionsItemEdge'
|
||||||
@ -717,7 +724,7 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
|
|||||||
playlist_title='%s - Collections' % channel_name)
|
playlist_title='%s - Collections' % channel_name)
|
||||||
|
|
||||||
|
|
||||||
class TwitchStreamIE(TwitchBaseIE):
|
class TwitchStreamIE(TwitchGraphQLBaseIE):
|
||||||
IE_NAME = 'twitch:stream'
|
IE_NAME = 'twitch:stream'
|
||||||
_VALID_URL = r'''(?x)
|
_VALID_URL = r'''(?x)
|
||||||
https?://
|
https?://
|
||||||
@ -774,28 +781,43 @@ class TwitchStreamIE(TwitchBaseIE):
|
|||||||
else super(TwitchStreamIE, cls).suitable(url))
|
else super(TwitchStreamIE, cls).suitable(url))
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
channel_name = self._match_id(url)
|
channel_name = self._match_id(url).lower()
|
||||||
|
|
||||||
access_token = self._download_access_token(channel_name)
|
gql = self._download_gql(
|
||||||
|
channel_name, [{
|
||||||
|
'operationName': 'StreamMetadata',
|
||||||
|
'variables': {'channelLogin': channel_name},
|
||||||
|
}, {
|
||||||
|
'operationName': 'ComscoreStreamingQuery',
|
||||||
|
'variables': {
|
||||||
|
'channel': channel_name,
|
||||||
|
'clipSlug': '',
|
||||||
|
'isClip': False,
|
||||||
|
'isLive': True,
|
||||||
|
'isVodOrCollection': False,
|
||||||
|
'vodID': '',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'operationName': 'VideoPreviewOverlay',
|
||||||
|
'variables': {'login': channel_name},
|
||||||
|
}],
|
||||||
|
'Downloading stream GraphQL')
|
||||||
|
|
||||||
token = access_token['token']
|
user = gql[0]['data']['user']
|
||||||
channel_id = self._extract_channel_id(token, channel_name)
|
|
||||||
|
|
||||||
stream = self._call_api(
|
if not user:
|
||||||
'kraken/streams/%s?stream_type=all' % channel_id,
|
raise ExtractorError(
|
||||||
channel_id, 'Downloading stream JSON').get('stream')
|
'%s does not exist' % channel_name, expected=True)
|
||||||
|
|
||||||
|
stream = user['stream']
|
||||||
|
|
||||||
if not stream:
|
if not stream:
|
||||||
raise ExtractorError('%s is offline' % channel_id, expected=True)
|
raise ExtractorError('%s is offline' % channel_name, expected=True)
|
||||||
|
|
||||||
# Channel name may be typed if different case than the original channel name
|
access_token = self._download_access_token(channel_name)
|
||||||
# (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
|
token = access_token['token']
|
||||||
# an invalid m3u8 URL. Working around by use of original channel name from stream
|
|
||||||
# JSON and fallback to lowercase if it's not available.
|
|
||||||
channel_name = try_get(
|
|
||||||
stream, lambda x: x['channel']['name'],
|
|
||||||
compat_str) or channel_name.lower()
|
|
||||||
|
|
||||||
|
stream_id = stream.get('id') or channel_name
|
||||||
query = {
|
query = {
|
||||||
'allow_source': 'true',
|
'allow_source': 'true',
|
||||||
'allow_audio_only': 'true',
|
'allow_audio_only': 'true',
|
||||||
@ -808,41 +830,39 @@ class TwitchStreamIE(TwitchBaseIE):
|
|||||||
'token': token.encode('utf-8'),
|
'token': token.encode('utf-8'),
|
||||||
}
|
}
|
||||||
formats = self._extract_m3u8_formats(
|
formats = self._extract_m3u8_formats(
|
||||||
'%s/api/channel/hls/%s.m3u8?%s'
|
'%s/api/channel/hls/%s.m3u8' % (self._USHER_BASE, channel_name),
|
||||||
% (self._USHER_BASE, channel_name, compat_urllib_parse_urlencode(query)),
|
stream_id, 'mp4', query=query)
|
||||||
channel_id, 'mp4')
|
|
||||||
self._prefer_source(formats)
|
self._prefer_source(formats)
|
||||||
|
|
||||||
view_count = stream.get('viewers')
|
view_count = stream.get('viewers')
|
||||||
timestamp = parse_iso8601(stream.get('created_at'))
|
timestamp = unified_timestamp(stream.get('createdAt'))
|
||||||
|
|
||||||
channel = stream['channel']
|
sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {}
|
||||||
title = self._live_title(channel.get('display_name') or channel.get('name'))
|
uploader = sq_user.get('displayName')
|
||||||
description = channel.get('status')
|
description = try_get(
|
||||||
|
sq_user, lambda x: x['broadcastSettings']['title'], compat_str)
|
||||||
|
|
||||||
thumbnails = []
|
thumbnail = url_or_none(try_get(
|
||||||
for thumbnail_key, thumbnail_url in stream['preview'].items():
|
gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'],
|
||||||
m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
|
compat_str))
|
||||||
if not m:
|
|
||||||
continue
|
title = uploader or channel_name
|
||||||
thumbnails.append({
|
stream_type = stream.get('type')
|
||||||
'url': thumbnail_url,
|
if stream_type in ['rerun', 'live']:
|
||||||
'width': int(m.group('width')),
|
title += ' (%s)' % stream_type
|
||||||
'height': int(m.group('height')),
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': str_or_none(stream.get('_id')) or channel_id,
|
'id': stream_id,
|
||||||
'display_id': channel_name,
|
'display_id': channel_name,
|
||||||
'title': title,
|
'title': self._live_title(title),
|
||||||
'description': description,
|
'description': description,
|
||||||
'thumbnails': thumbnails,
|
'thumbnail': thumbnail,
|
||||||
'uploader': channel.get('display_name'),
|
'uploader': uploader,
|
||||||
'uploader_id': channel.get('name'),
|
'uploader_id': channel_name,
|
||||||
'timestamp': timestamp,
|
'timestamp': timestamp,
|
||||||
'view_count': view_count,
|
'view_count': view_count,
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
'is_live': True,
|
'is_live': stream_type == 'live',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user