mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-09-06 02:49:35 +02:00
InfoExtractor: add some helper methods to extract OpenGraph info
This commit is contained in:
parent
d8269e1dfb
commit
46720279c2
@ -257,6 +257,30 @@ def _get_login_info(self):
|
||||
|
||||
return (username, password)
|
||||
|
||||
# Helper functions for extracting OpenGraph info
|
||||
@staticmethod
|
||||
def _og_regex(property):
|
||||
return r'<meta.+?property=[\'"]og:%s[\'"].+?content=(?:"(.+?)"|\'(.+?)\')' % property
|
||||
|
||||
def _og_search_property(self, property, html, name=None, **kargs):
|
||||
if name is None:
|
||||
name = 'OpenGraph %s' % property
|
||||
return self._html_search_regex(self._og_regex(property), html, name, **kargs)
|
||||
|
||||
def _og_search_thumbnail(self, html, **kargs):
|
||||
return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
|
||||
|
||||
def _og_search_description(self, html, **kargs):
|
||||
return self._og_search_property('description', html, fatal=False, **kargs)
|
||||
|
||||
def _og_search_title(self, html, **kargs):
|
||||
return self._og_search_property('title', html, **kargs)
|
||||
|
||||
def _og_search_video_url(self, html, name='video url', **kargs):
|
||||
return self._html_search_regex([self._og_regex('video:secure_url'),
|
||||
self._og_regex('video')],
|
||||
html, name, **kargs)
|
||||
|
||||
class SearchInfoExtractor(InfoExtractor):
|
||||
"""
|
||||
Base class for paged search queries extractors.
|
||||
|
@ -34,8 +34,6 @@ def _real_extract(self, url):
|
||||
description = self._html_search_regex(r'<meta (?:property="og:|name=")description" content="(.*?)"',
|
||||
webpage, 'description',
|
||||
flags=re.MULTILINE|re.DOTALL)
|
||||
thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.*?)"',
|
||||
webpage, 'thumbnail')
|
||||
|
||||
url = self._search_regex(r'<string name="URL">(.*?)</string>',
|
||||
video_info, 'video url')
|
||||
@ -49,5 +47,5 @@ def _real_extract(self, url):
|
||||
'url': url,
|
||||
'play_path': path,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
}
|
||||
|
@ -39,9 +39,6 @@ def _real_extract(self, url):
|
||||
# Extract URL, uploader and title from webpage
|
||||
self.report_extraction(video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<meta property="og:title" content="(.*?)" />',
|
||||
webpage, 'title')
|
||||
|
||||
video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
|
||||
# Looking for official user
|
||||
r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
|
||||
@ -76,7 +73,7 @@ def _real_extract(self, url):
|
||||
'url': video_url,
|
||||
'uploader': video_uploader,
|
||||
'upload_date': video_upload_date,
|
||||
'title': video_title,
|
||||
'title': self._og_search_title(webpage),
|
||||
'ext': video_extension,
|
||||
'thumbnail': info['thumbnail_url']
|
||||
}]
|
||||
|
@ -28,14 +28,9 @@ def _real_extract(self, url):
|
||||
video_url = self._search_regex(r'(?:file|source)=(http[^\'"&]*)',
|
||||
webpage, u'video URL')
|
||||
final_url = compat_urllib_parse.unquote(video_url)
|
||||
thumbnail_url = self._search_regex(r'<meta property="og:image" content="(.+?)" />',
|
||||
webpage, u'thumbnail URL')
|
||||
uploader = self._search_regex(r'<meta name="uploader" content="(.+?)" />',
|
||||
webpage, u'uploader')
|
||||
title = self._search_regex(r'<meta property="og:title" content="(.+?)" />',
|
||||
webpage, u'Video title').replace(' | eHow', '')
|
||||
description = self._search_regex(r'<meta property="og:description" content="(.+?)" />',
|
||||
webpage, u'video description')
|
||||
title = self._og_search_title(webpage).replace(' | eHow', '')
|
||||
ext = determine_ext(final_url)
|
||||
|
||||
return {
|
||||
@ -44,8 +39,8 @@ def _real_extract(self, url):
|
||||
'url': final_url,
|
||||
'ext': ext,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail_url,
|
||||
'description': description,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'description': self._og_search_description(webpage),
|
||||
'uploader': uploader,
|
||||
}
|
||||
|
||||
|
@ -36,11 +36,7 @@ def _real_extract(self, url):
|
||||
videoDesc = self._html_search_regex('<meta name="description" content="([^"]*)"',
|
||||
webpage, u'description', fatal=False)
|
||||
|
||||
imgUrl = self._html_search_regex('<meta property="og:image" content="([^"]*)"',
|
||||
webpage, u'thumbnail', fatal=False)
|
||||
|
||||
playerUrl = self._html_search_regex('<meta property="og:video" content="([^"]*)"',
|
||||
webpage, u'player url')
|
||||
playerUrl = self._og_search_video_url(webpage, name='player url')
|
||||
|
||||
title = self._html_search_regex('<meta name="title" content="([^"]*)"',
|
||||
webpage, u'player url').split(' : ')[-1]
|
||||
@ -70,7 +66,7 @@ def _real_extract(self, url):
|
||||
'upload_date': None,
|
||||
'title': title,
|
||||
'ext': 'mp4',
|
||||
'thumbnail': imgUrl,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'description': videoDesc,
|
||||
'player_url': playerUrl,
|
||||
}
|
||||
|
@ -47,21 +47,12 @@ def _real_extract(self, url):
|
||||
raise ExtractorError(u'Unable to extract video url')
|
||||
video_url = mobj.group(1) + unescapeHTML(mobj.group(2))
|
||||
|
||||
video_title = self._html_search_regex(r'<meta property="og:title" content=(?:"([^"]+)"|\'([^\']+)\')',
|
||||
webpage, u'video title')
|
||||
|
||||
video_description = self._html_search_regex(r'<meta property="og:description" content=(?:"([^"]+)"|\'([^\']+)\')',
|
||||
webpage, u'description', fatal=False)
|
||||
|
||||
thumbnail = self._html_search_regex(r'<meta property="og:image" content=(?:"([^"]+)"|\'([^\']+)\')',
|
||||
webpage, u'thumbnail', fatal=False)
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'mp4',
|
||||
'title': video_title,
|
||||
'description': video_description,
|
||||
'thumbnail': thumbnail,
|
||||
'title': self._og_search_title(webpage),
|
||||
'description': self._og_search_description(webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'uploader_id': video_uploader_id,
|
||||
}]
|
||||
|
@ -27,14 +27,11 @@ def _real_extract(self, url):
|
||||
title = self._html_search_regex((r"<h1 class='player_page_h1'.*?>(?P<title>.*?)</h1>",
|
||||
r'<title>(?P<title>[^<]+?)</title>'), webpage, 'title', flags=re.DOTALL)
|
||||
|
||||
video_description = self._html_search_regex(r'<meta property="og:description" content="(?P<desc>.*?)"',
|
||||
webpage, u'description', fatal=False, flags=re.DOTALL)
|
||||
|
||||
info = {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'mp4',
|
||||
'title': title,
|
||||
'description': video_description,
|
||||
'description': self._og_search_description(webpage, flags=re.DOTALL),
|
||||
}
|
||||
return [info]
|
||||
|
@ -33,16 +33,12 @@ def _real_extract(self, url):
|
||||
|
||||
video_title = self._html_search_regex(r"<title>(.*)</title>",
|
||||
webpage_src, u'title')
|
||||
|
||||
# Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
|
||||
thumbnail = self._html_search_regex(r'"og:image" content="(.*)"',
|
||||
webpage_src, u'thumbnail', fatal=False)
|
||||
|
||||
results = [{
|
||||
'id': video_id,
|
||||
'url' : video_url,
|
||||
'title' : video_title,
|
||||
'thumbnail' : thumbnail,
|
||||
'thumbnail' : self._og_search_thumbnail(webpage_src),
|
||||
'ext' : 'mp3',
|
||||
}]
|
||||
return results
|
||||
return results
|
||||
|
@ -18,12 +18,6 @@ def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group(1)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_url = self._html_search_regex(
|
||||
r'<meta property="og:video" content="(.+?)"',
|
||||
webpage, u'video URL')
|
||||
thumbnail_url = self._html_search_regex(
|
||||
r'<meta property="og:image" content="(.+?)" />',
|
||||
webpage, u'thumbnail URL', fatal=False)
|
||||
html_title = self._html_search_regex(
|
||||
r'<title>(.+?)</title>',
|
||||
webpage, u'title', flags=re.DOTALL)
|
||||
@ -34,9 +28,9 @@ def _real_extract(self, url):
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'url': self._og_search_video_url(webpage),
|
||||
'ext': ext,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail_url,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'uploader_id' : uploader_id
|
||||
}]
|
||||
|
@ -24,8 +24,7 @@ def _real_extract(self, url):
|
||||
thumbnail = u'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<meta property="og:title" content="(?P<title>.*?)"',
|
||||
webpage, u'title')
|
||||
video_title = self._og_search_title(webpage)
|
||||
|
||||
uploader = self._html_search_regex(r'<div class="user-name-and-bio">[\S\s]+?<h2>(?P<uploader>.+?)</h2>',
|
||||
webpage, u'uploader', fatal=False)
|
||||
|
@ -33,11 +33,9 @@ def _real_extract(self, url):
|
||||
video_url = self._search_regex(r'file: "(.*?)",',
|
||||
webpage, u'video URL')
|
||||
|
||||
video_title = self._html_search_regex(r'<meta property="og:title" content="(?P<title>.*?)"',
|
||||
webpage, u'title').replace('LiveLeak.com -', '').strip()
|
||||
video_title = self._og_search_title(webpage).replace('LiveLeak.com -', '').strip()
|
||||
|
||||
video_description = self._html_search_regex(r'<meta property="og:description" content="(?P<desc>.*?)"',
|
||||
webpage, u'description', fatal=False)
|
||||
video_description = self._og_search_description(webpage)
|
||||
|
||||
video_uploader = self._html_search_regex(r'By:.*?(\w+)</a>',
|
||||
webpage, u'uploader', fatal=False)
|
||||
|
@ -30,8 +30,7 @@ def _real_extract(self, url):
|
||||
video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
|
||||
|
||||
shortened_video_id = video_id.rpartition('/')[2]
|
||||
title = self._html_search_regex(r'<meta property="og:title" content="(.*?)"',
|
||||
webpage, 'title', default=shortened_video_id).replace('NBA.com: ', '')
|
||||
title = self._og_search_title(webpage, default=shortened_video_id).replace('NBA.com: ', '')
|
||||
|
||||
# It isn't there in the HTML it returns to us
|
||||
# uploader_date = self._html_search_regex(r'<b>Date:</b> (.*?)</div>', webpage, 'upload_date', fatal=False)
|
||||
|
@ -18,12 +18,6 @@ def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group(1)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_url = self._html_search_regex(
|
||||
r'<meta property="og:video:secure_url" content="(.+?)">',
|
||||
webpage, u'video URL')
|
||||
thumbnail_url = self._html_search_regex(
|
||||
r'<meta property="og:image" content="(.+?)" />',
|
||||
webpage, u'thumbnail URL', fatal=False)
|
||||
html_title = self._html_search_regex(
|
||||
r'<title>(.+?)</title>',
|
||||
webpage, u'title')
|
||||
@ -34,9 +28,9 @@ def _real_extract(self, url):
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'url': self._og_search_video_url(webpage),
|
||||
'ext': ext,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail_url,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'uploader_id' : uploader_id
|
||||
}]
|
||||
|
@ -30,15 +30,6 @@ def _real_extract(self, url):
|
||||
|
||||
self.report_extraction(video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<meta property="og:title" content="(.+?)"',
|
||||
webpage, u'title')
|
||||
|
||||
thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)"',
|
||||
webpage, u'thumbnail', fatal=False)
|
||||
|
||||
video_description = self._html_search_regex(r'<meta property="og:description" content="(.*?)"',
|
||||
webpage, u'description', fatal=False)
|
||||
|
||||
data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
|
||||
data = self._download_webpage(data_url, video_id, 'Downloading data webpage')
|
||||
|
||||
@ -49,7 +40,7 @@ def _real_extract(self, url):
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'mp4',
|
||||
'title': video_title,
|
||||
'thumbnail': thumbnail,
|
||||
'description': video_description,
|
||||
'title': self._og_search_title(webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'description': self._og_search_description(webpage),
|
||||
}]
|
||||
|
@ -24,11 +24,8 @@ def _real_extract(self, url):
|
||||
webpage, 'video title').replace(' - Trailer Addict','')
|
||||
view_count = self._search_regex(r'Views: (.+?)<br />',
|
||||
webpage, 'Views Count')
|
||||
description = self._search_regex(r'<meta property="og:description" content="(.+?)" />',
|
||||
webpage, 'video description')
|
||||
video_id = self._search_regex(r'<meta property="og:video" content="(.+?)" />',
|
||||
webpage, 'Video id').split('=')[1]
|
||||
|
||||
video_id = self._og_search_property('video', webpage, 'Video id').split('=')[1]
|
||||
|
||||
info_url = "http://www.traileraddict.com/fvar.php?tid=%s" %(str(video_id))
|
||||
info_webpage = self._download_webpage(info_url, video_id , "Downloading the info webpage")
|
||||
|
||||
@ -44,6 +41,6 @@ def _real_extract(self, url):
|
||||
'ext' : ext,
|
||||
'title' : title,
|
||||
'thumbnail' : thumbnail_url,
|
||||
'description' : description,
|
||||
'description' : self._og_search_description(webpage),
|
||||
'view_count' : view_count,
|
||||
}]
|
||||
|
@ -22,8 +22,6 @@ def _real_extract(self, url):
|
||||
video_id = mobj.group('id')
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
title = self._html_search_regex(
|
||||
r'<meta property="og:title" content="(.*?)">', webpage, u'title')
|
||||
internal_id = self._search_regex(r'codVideo=([0-9]+)', webpage, u'internal video ID')
|
||||
|
||||
data_url = u'http://tu.tv/flvurl.php?codVideo=' + str(internal_id)
|
||||
@ -36,6 +34,6 @@ def _real_extract(self, url):
|
||||
'id': internal_id,
|
||||
'url': video_url,
|
||||
'ext': ext,
|
||||
'title': title,
|
||||
'title': self._og_search_title(webpage),
|
||||
}
|
||||
return [info]
|
||||
|
@ -27,12 +27,6 @@ def _real_extract(self, url):
|
||||
video_url = self._html_search_regex(r'<meta property="twitter:player:stream" content="(.+?)"',
|
||||
webpage, u'video URL')
|
||||
|
||||
video_title = self._html_search_regex(r'<meta property="og:title" content="(.+?)"',
|
||||
webpage, u'title')
|
||||
|
||||
thumbnail = self._html_search_regex(r'<meta property="og:image" content="(.+?)(\?.*?)?"',
|
||||
webpage, u'thumbnail', fatal=False)
|
||||
|
||||
uploader = self._html_search_regex(r'<div class="user">.*?<h2>(.+?)</h2>',
|
||||
webpage, u'uploader', fatal=False, flags=re.DOTALL)
|
||||
|
||||
@ -40,7 +34,7 @@ def _real_extract(self, url):
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'mp4',
|
||||
'title': video_title,
|
||||
'thumbnail': thumbnail,
|
||||
'title': self._og_search_title(webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'uploader': uploader,
|
||||
}]
|
||||
|
Loading…
Reference in New Issue
Block a user