mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-11-02 17:22:31 +01:00
Merge pull request #563 from FiloSottile/IE_cleanup
General IE docs and return dicts cleanup
This commit is contained in:
Filippo Valsorda
commit
e643e2c6b7
@ -327,10 +327,13 @@ def prepare_filename(self, info_dict):
|
||||
"""Generate the output filename."""
|
||||
try:
|
||||
template_dict = dict(info_dict)
|
||||
|
||||
template_dict['epoch'] = int(time.time())
|
||||
template_dict['autonumber'] = u'%05d' % self._num_downloads
|
||||
|
||||
template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
|
||||
template_dict = dict((k, sanitize_filename(u(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
|
||||
|
||||
filename = self.params['outtmpl'] % template_dict
|
||||
return filename
|
||||
except (ValueError, KeyError), err:
|
||||
@ -359,6 +362,9 @@ def process_info(self, info_dict):
|
||||
# Keep for backwards compatibility
|
||||
info_dict['stitle'] = info_dict['title']
|
||||
|
||||
if not 'format' in info_dict:
|
||||
info_dict['format'] = info_dict['ext']
|
||||
|
||||
reason = self._match_entry(info_dict)
|
||||
if reason is not None:
|
||||
self.to_screen(u'[download] ' + reason)
|
||||
@ -481,6 +487,11 @@ def download(self, url_list):
|
||||
if not ie.suitable(url):
|
||||
continue
|
||||
|
||||
# Warn if the _WORKING attribute is False
|
||||
if not ie.working():
|
||||
self.trouble(u'WARNING: the program functionality for this site has been marked as broken, '
|
||||
u'and will probably not work. If you want to go on, use the -i option.')
|
||||
|
||||
# Suitable InfoExtractor found
|
||||
suitable_found = True
|
||||
|
||||
|
||||
@ -29,37 +29,48 @@ class InfoExtractor(object):
|
||||
"""Information Extractor class.
|
||||
|
||||
Information extractors are the classes that, given a URL, extract
|
||||
information from the video (or videos) the URL refers to. This
|
||||
information includes the real video URL, the video title and simplified
|
||||
title, author and others. The information is stored in a dictionary
|
||||
which is then passed to the FileDownloader. The FileDownloader
|
||||
processes this information possibly downloading the video to the file
|
||||
system, among other possible outcomes. The dictionaries must include
|
||||
the following fields:
|
||||
information about the video (or videos) the URL refers to. This
|
||||
information includes the real video URL, the video title, author and
|
||||
others. The information is stored in a dictionary which is then
|
||||
passed to the FileDownloader. The FileDownloader processes this
|
||||
information possibly downloading the video to the file system, among
|
||||
other possible outcomes.
|
||||
|
||||
The dictionaries must include the following fields:
|
||||
|
||||
id: Video identifier.
|
||||
url: Final video URL.
|
||||
uploader: Nickname of the video uploader.
|
||||
title: Literal title.
|
||||
uploader: Nickname of the video uploader, unescaped.
|
||||
upload_date: Video upload date (YYYYMMDD).
|
||||
title: Video title, unescaped.
|
||||
ext: Video filename extension.
|
||||
format: Video format.
|
||||
player_url: SWF Player URL (may be None).
|
||||
|
||||
The following fields are optional. Their primary purpose is to allow
|
||||
youtube-dl to serve as the backend for a video search function, such
|
||||
as the one in youtube2mp3. They are only used when their respective
|
||||
forced printing functions are called:
|
||||
The following fields are optional:
|
||||
|
||||
format: The video format, defaults to ext (used for --get-format)
|
||||
thumbnail: Full URL to a video thumbnail image.
|
||||
description: One-line video description.
|
||||
player_url: SWF Player URL (used for rtmpdump).
|
||||
subtitles: The .srt file contents.
|
||||
urlhandle: [internal] The urlHandle to be used to download the file,
|
||||
like returned by urllib2.urlopen
|
||||
|
||||
The fields should all be Unicode strings.
|
||||
|
||||
Subclasses of this one should re-define the _real_initialize() and
|
||||
_real_extract() methods and define a _VALID_URL regexp.
|
||||
Probably, they should also be added to the list of extractors.
|
||||
|
||||
_real_extract() must return a *list* of information dictionaries as
|
||||
described above.
|
||||
|
||||
Finally, the _WORKING attribute should be set to False for broken IEs
|
||||
in order to warn the users and skip the tests.
|
||||
"""
|
||||
|
||||
_ready = False
|
||||
_downloader = None
|
||||
_WORKING = True
|
||||
|
||||
def __init__(self, downloader=None):
|
||||
"""Constructor. Receives an optional downloader."""
|
||||
@ -70,6 +81,10 @@ def suitable(self, url):
|
||||
"""Receives a URL and returns True if suitable for this IE."""
|
||||
return re.match(self._VALID_URL, url) is not None
|
||||
|
||||
def working(self):
|
||||
"""Getter method for _WORKING."""
|
||||
return self._WORKING
|
||||
|
||||
def initialize(self):
|
||||
"""Initializes an instance (authentication, etc)."""
|
||||
if not self._ready:
|
||||
@ -365,7 +380,7 @@ def _real_extract(self, url):
|
||||
video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
|
||||
|
||||
# upload date
|
||||
upload_date = u'NA'
|
||||
upload_date = None
|
||||
mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
|
||||
if mobj is not None:
|
||||
upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
|
||||
@ -475,6 +490,9 @@ def _real_extract(self, url):
|
||||
# Extension
|
||||
video_extension = self._video_extensions.get(format_param, 'flv')
|
||||
|
||||
video_format = '{} - {}'.format(format_param.decode('utf-8') if format_param else video_extension.decode('utf-8'),
|
||||
self._video_dimensions.get(format_param, '???'))
|
||||
|
||||
results.append({
|
||||
'id': video_id.decode('utf-8'),
|
||||
'url': video_real_url.decode('utf-8'),
|
||||
@ -482,7 +500,7 @@ def _real_extract(self, url):
|
||||
'upload_date': upload_date,
|
||||
'title': video_title,
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
|
||||
'format': video_format,
|
||||
'thumbnail': video_thumbnail.decode('utf-8'),
|
||||
'description': video_description,
|
||||
'player_url': player_url,
|
||||
@ -613,11 +631,9 @@ def _real_extract(self, url):
|
||||
'id': video_id.decode('utf-8'),
|
||||
'url': video_url.decode('utf-8'),
|
||||
'uploader': video_uploader.decode('utf-8'),
|
||||
'upload_date': u'NA',
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
@ -691,7 +707,7 @@ def _real_extract(self, url):
|
||||
return
|
||||
video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
|
||||
|
||||
video_uploader = u'NA'
|
||||
video_uploader = None
|
||||
mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
|
||||
if mobj is None:
|
||||
# lookin for official user
|
||||
@ -703,7 +719,7 @@ def _real_extract(self, url):
|
||||
else:
|
||||
video_uploader = mobj.group(1)
|
||||
|
||||
video_upload_date = u'NA'
|
||||
video_upload_date = None
|
||||
mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
|
||||
if mobj is not None:
|
||||
video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
|
||||
@ -715,8 +731,6 @@ def _real_extract(self, url):
|
||||
'upload_date': video_upload_date,
|
||||
'title': video_title,
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
@ -806,12 +820,10 @@ def _real_extract(self, url):
|
||||
return [{
|
||||
'id': video_id.decode('utf-8'),
|
||||
'url': video_url.decode('utf-8'),
|
||||
'uploader': u'NA',
|
||||
'upload_date': u'NA',
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
@ -874,11 +886,9 @@ def _real_extract(self, url):
|
||||
'id': video_id.decode('utf-8'),
|
||||
'url': video_url.decode('utf-8'),
|
||||
'uploader': video_uploader,
|
||||
'upload_date': u'NA',
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
@ -1016,13 +1026,11 @@ def _real_extract(self, url, new_video=True):
|
||||
'id': video_id.decode('utf-8'),
|
||||
'url': video_url,
|
||||
'uploader': video_uploader,
|
||||
'upload_date': u'NA',
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'thumbnail': video_thumbnail.decode('utf-8'),
|
||||
'description': video_description,
|
||||
'thumbnail': video_thumbnail,
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
@ -1090,7 +1098,7 @@ def _real_extract(self, url, new_video=True):
|
||||
else: video_description = ''
|
||||
|
||||
# Extract upload date
|
||||
video_upload_date = u'NA'
|
||||
video_upload_date = None
|
||||
mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
|
||||
if mobj is not None:
|
||||
video_upload_date = mobj.group(1)
|
||||
@ -1136,7 +1144,6 @@ def _real_extract(self, url, new_video=True):
|
||||
'ext': video_extension,
|
||||
'thumbnail': video_thumbnail,
|
||||
'description': video_description,
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
@ -1416,11 +1423,9 @@ def _real_extract(self, url):
|
||||
'id': video_id.decode('utf-8'),
|
||||
'url': video_url.decode('utf-8'),
|
||||
'uploader': video_uploader,
|
||||
'upload_date': u'NA',
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
@ -2021,18 +2026,17 @@ def _real_extract(self, url):
|
||||
return [{
|
||||
'id': file_id.decode('utf-8'),
|
||||
'url': file_url.decode('utf-8'),
|
||||
'uploader': u'NA',
|
||||
'upload_date': u'NA',
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
'title': file_title,
|
||||
'ext': file_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
|
||||
class FacebookIE(InfoExtractor):
|
||||
"""Information Extractor for Facebook"""
|
||||
|
||||
_WORKING = False
|
||||
_VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
|
||||
_LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
|
||||
_NETRC_MACHINE = 'facebook'
|
||||
@ -2177,7 +2181,7 @@ def _real_extract(self, url):
|
||||
video_thumbnail = video_info['thumbnail']
|
||||
|
||||
# upload date
|
||||
upload_date = u'NA'
|
||||
upload_date = None
|
||||
if 'upload_date' in video_info:
|
||||
upload_time = video_info['upload_date']
|
||||
timetuple = email.utils.parsedate_tz(upload_time)
|
||||
@ -2232,7 +2236,6 @@ def _real_extract(self, url):
|
||||
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
|
||||
'thumbnail': video_thumbnail.decode('utf-8'),
|
||||
'description': video_description.decode('utf-8'),
|
||||
'player_url': None,
|
||||
})
|
||||
return results
|
||||
|
||||
@ -2276,6 +2279,8 @@ def _real_extract(self, url):
|
||||
info = {
|
||||
'id': title,
|
||||
'url': url,
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
'title': title,
|
||||
'ext': ext,
|
||||
'urlhandle': urlh
|
||||
@ -2376,12 +2381,10 @@ def _real_extract(self,url):
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'uploader': u'NA',
|
||||
'upload_date': u'NA',
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': u'flv',
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
class ComedyCentralIE(InfoExtractor):
|
||||
@ -2638,7 +2641,6 @@ def _real_extract(self, url):
|
||||
'upload_date': None,
|
||||
'title': showName,
|
||||
'ext': 'flv',
|
||||
'format': 'flv',
|
||||
'thumbnail': imgUrl,
|
||||
'description': description,
|
||||
'player_url': playerUrl,
|
||||
@ -2685,6 +2687,8 @@ def _real_extract(self, url):
|
||||
info = {
|
||||
'id': video_id,
|
||||
'internal_id': internal_video_id,
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
}
|
||||
|
||||
self.report_extraction(video_id)
|
||||
@ -2703,7 +2707,6 @@ def _real_extract(self, url):
|
||||
info['url'] = videoNode.findall('./file')[0].text
|
||||
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
|
||||
info['ext'] = info['url'].rpartition('.')[2]
|
||||
info['format'] = info['ext']
|
||||
except IndexError:
|
||||
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
|
||||
return
|
||||
@ -2774,10 +2777,8 @@ def _real_extract(self, url):
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': 'flv',
|
||||
'format': 'flv',
|
||||
'thumbnail': video_thumbnail,
|
||||
'description': None,
|
||||
'player_url': None,
|
||||
}
|
||||
|
||||
return [info]
|
||||
@ -2871,8 +2872,6 @@ def _real_extract(self, url):
|
||||
'upload_date': upload_date,
|
||||
'title': title,
|
||||
'ext': u'mp3',
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
'description': description.decode('utf-8')
|
||||
}]
|
||||
|
||||
@ -2939,11 +2938,9 @@ def _real_extract(self, url):
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': extension,
|
||||
'format': extension, # Extension is always(?) mp4, but seems to be flv
|
||||
'ext': extension, # Extension is always(?) mp4, but seems to be flv
|
||||
'thumbnail': None,
|
||||
'description': video_description,
|
||||
'player_url': None,
|
||||
}
|
||||
|
||||
return [info]
|
||||
@ -3052,7 +3049,7 @@ def _real_extract(self, url):
|
||||
'id': file_id.decode('utf-8'),
|
||||
'url': file_url.decode('utf-8'),
|
||||
'uploader': uploader.decode('utf-8'),
|
||||
'upload_date': u'NA',
|
||||
'upload_date': None,
|
||||
'title': json_data['name'],
|
||||
'ext': file_url.split('.')[-1].decode('utf-8'),
|
||||
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
|
||||
@ -3086,6 +3083,8 @@ def _real_extract(self, url):
|
||||
video = mobj.group('video')
|
||||
info = {
|
||||
'id': course + '_' + video,
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
}
|
||||
|
||||
self.report_extraction(info['id'])
|
||||
@ -3104,13 +3103,14 @@ def _real_extract(self, url):
|
||||
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
|
||||
return
|
||||
info['ext'] = info['url'].rpartition('.')[2]
|
||||
info['format'] = info['ext']
|
||||
return [info]
|
||||
elif mobj.group('course'): # A course page
|
||||
course = mobj.group('course')
|
||||
info = {
|
||||
'id': course,
|
||||
'type': 'playlist',
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
}
|
||||
|
||||
self.report_download_webpage(info['id'])
|
||||
@ -3147,6 +3147,8 @@ def _real_extract(self, url):
|
||||
info = {
|
||||
'id': 'Stanford OpenClassroom',
|
||||
'type': 'playlist',
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
}
|
||||
|
||||
self.report_download_webpage(info['id'])
|
||||
@ -3255,6 +3257,7 @@ def _real_extract(self, url):
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'uploader': performer,
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': ext,
|
||||
'format': format,
|
||||
@ -3376,9 +3379,9 @@ def _real_extract(self, url):
|
||||
'id': '%s_part%02d' % (video_id, index),
|
||||
'url': download_url,
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': ext,
|
||||
'format': u'NA'
|
||||
}
|
||||
files_info.append(info)
|
||||
|
||||
@ -3436,18 +3439,16 @@ def _real_extract(self, url):
|
||||
return
|
||||
video_thumbnail = result.group(1).decode('utf-8')
|
||||
|
||||
info = {'id': video_id,
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'uploader': None,
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': 'flv',
|
||||
'format': 'flv',
|
||||
'thumbnail': video_thumbnail,
|
||||
'description': None,
|
||||
'player_url': None}
|
||||
|
||||
return [info]
|
||||
}]
|
||||
|
||||
|
||||
class GooglePlusIE(InfoExtractor):
|
||||
@ -3501,7 +3502,7 @@ def _real_extract(self, url):
|
||||
return
|
||||
|
||||
# Extract update date
|
||||
upload_date = u'NA'
|
||||
upload_date = None
|
||||
pattern = 'title="Timestamp">(.*?)</a>'
|
||||
mobj = re.search(pattern, webpage)
|
||||
if mobj:
|
||||
@ -3512,7 +3513,7 @@ def _real_extract(self, url):
|
||||
self.report_date(upload_date)
|
||||
|
||||
# Extract uploader
|
||||
uploader = u'NA'
|
||||
uploader = None
|
||||
pattern = r'rel\="author".*?>(.*?)</a>'
|
||||
mobj = re.search(pattern, webpage)
|
||||
if mobj:
|
||||
@ -3569,6 +3570,4 @@ def _real_extract(self, url):
|
||||
'upload_date': upload_date.decode('utf-8'),
|
||||
'title': video_title.decode('utf-8'),
|
||||
'ext': video_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
}]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user