mirror of
https://github.com/blackjack4494/yt-dlc.git
synced 2024-08-18 16:29:43 +02:00
Merge 'rg3/master' into fork_master
This commit is contained in:
commit
3c6ffbaedb
@ -51,7 +51,10 @@ which means you can modify it, redistribute it or use it however you like.
|
|||||||
%(upload_date)s for the upload date (YYYYMMDD),
|
%(upload_date)s for the upload date (YYYYMMDD),
|
||||||
%(extractor)s for the provider (youtube, metacafe,
|
%(extractor)s for the provider (youtube, metacafe,
|
||||||
etc), %(id)s for the video id and %% for a literal
|
etc), %(id)s for the video id and %% for a literal
|
||||||
percent. Use - to output to stdout.
|
percent. Use - to output to stdout. Can also be
|
||||||
|
used to download to a different directory, for
|
||||||
|
example with -o '/my/downloads/%(uploader)s/%(title
|
||||||
|
)s-%(id)s.%(ext)s' .
|
||||||
--restrict-filenames Restrict filenames to only ASCII characters, and
|
--restrict-filenames Restrict filenames to only ASCII characters, and
|
||||||
avoid "&" and spaces in filenames
|
avoid "&" and spaces in filenames
|
||||||
-a, --batch-file FILE file containing URLs to download ('-' for stdin)
|
-a, --batch-file FILE file containing URLs to download ('-' for stdin)
|
||||||
@ -194,3 +197,5 @@ Please include:
|
|||||||
* The output of `youtube-dl --version`
|
* The output of `youtube-dl --version`
|
||||||
* The output of `python --version`
|
* The output of `python --version`
|
||||||
* The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
|
* The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
|
||||||
|
|
||||||
|
For discussions, join us in the irc channel #youtube-dl on freenode.
|
||||||
|
@ -34,31 +34,19 @@ import youtube_dl.InfoExtractors
|
|||||||
def _file_md5(fn):
|
def _file_md5(fn):
|
||||||
with open(fn, 'rb') as f:
|
with open(fn, 'rb') as f:
|
||||||
return hashlib.md5(f.read()).hexdigest()
|
return hashlib.md5(f.read()).hexdigest()
|
||||||
|
|
||||||
def md5_for_file(filename, block_size=2**20):
|
|
||||||
with open(filename) as f:
|
|
||||||
md5 = hashlib.md5()
|
|
||||||
while True:
|
|
||||||
data = f.read(block_size)
|
|
||||||
if not data:
|
|
||||||
break
|
|
||||||
md5.update(data)
|
|
||||||
return md5.hexdigest()
|
|
||||||
_file_md5 = md5_for_file
|
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
_skip_unless = unittest.skipUnless
|
_skip_unless = unittest.skipUnless
|
||||||
except AttributeError: # Python 2.6
|
except AttributeError: # Python 2.6
|
||||||
def _skip_unless(cond, reason='No reason given'):
|
def _skip_unless(cond, reason='No reason given'):
|
||||||
def resfunc(f):
|
def resfunc(f):
|
||||||
def wfunc(*args, **kwargs):
|
# Start the function name with test to appease nosetests-2.6
|
||||||
|
def test_wfunc(*args, **kwargs):
|
||||||
if cond:
|
if cond:
|
||||||
return f(*args, **kwargs)
|
return f(*args, **kwargs)
|
||||||
else:
|
else:
|
||||||
print('Skipped test')
|
print('Skipped test')
|
||||||
return
|
return
|
||||||
return wfunc
|
return test_wfunc
|
||||||
return resfunc
|
return resfunc
|
||||||
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
|
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
|
||||||
|
|
||||||
|
@ -20,31 +20,19 @@ import youtube_dl.InfoExtractors
|
|||||||
def _file_md5(fn):
|
def _file_md5(fn):
|
||||||
with open(fn, 'rb') as f:
|
with open(fn, 'rb') as f:
|
||||||
return hashlib.md5(f.read()).hexdigest()
|
return hashlib.md5(f.read()).hexdigest()
|
||||||
|
|
||||||
def md5_for_file(filename, block_size=2**20):
|
|
||||||
with open(filename) as f:
|
|
||||||
md5 = hashlib.md5()
|
|
||||||
while True:
|
|
||||||
data = f.read(block_size)
|
|
||||||
if not data:
|
|
||||||
break
|
|
||||||
md5.update(data)
|
|
||||||
return md5.hexdigest()
|
|
||||||
_file_md5 = md5_for_file
|
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
_skip_unless = unittest.skipUnless
|
_skip_unless = unittest.skipUnless
|
||||||
except AttributeError: # Python 2.6
|
except AttributeError: # Python 2.6
|
||||||
def _skip_unless(cond, reason='No reason given'):
|
def _skip_unless(cond, reason='No reason given'):
|
||||||
def resfunc(f):
|
def resfunc(f):
|
||||||
def wfunc(*args, **kwargs):
|
# Start the function name with test to appease nosetests-2.6
|
||||||
|
def test_wfunc(*args, **kwargs):
|
||||||
if cond:
|
if cond:
|
||||||
return f(*args, **kwargs)
|
return f(*args, **kwargs)
|
||||||
else:
|
else:
|
||||||
print('Skipped test')
|
print('Skipped test')
|
||||||
return
|
return
|
||||||
return wfunc
|
return test_wfunc
|
||||||
return resfunc
|
return resfunc
|
||||||
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
|
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
|
||||||
|
|
||||||
@ -79,7 +67,7 @@ class DownloadTest(unittest.TestCase):
|
|||||||
|
|
||||||
@_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
|
@_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
|
||||||
def test_Metacafe(self):
|
def test_Metacafe(self):
|
||||||
filename = 'aUehQsCQtM.flv'
|
filename = '_aUehQsCQtM.flv'
|
||||||
fd = FileDownloader(self.parameters)
|
fd = FileDownloader(self.parameters)
|
||||||
fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
|
fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
|
||||||
fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
|
fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
|
||||||
@ -120,7 +108,7 @@ class DownloadTest(unittest.TestCase):
|
|||||||
|
|
||||||
@_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
|
@_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
|
||||||
def test_Soundcloud(self):
|
def test_Soundcloud(self):
|
||||||
filename = 'n6FLbx6ZzMiu.mp3'
|
filename = '62986583.mp3'
|
||||||
fd = FileDownloader(self.parameters)
|
fd = FileDownloader(self.parameters)
|
||||||
fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
|
fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
|
||||||
fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
|
fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
|
||||||
@ -159,26 +147,38 @@ class DownloadTest(unittest.TestCase):
|
|||||||
md5_for_file = _file_md5(filename)
|
md5_for_file = _file_md5(filename)
|
||||||
self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c')
|
self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c')
|
||||||
|
|
||||||
|
@_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING")
|
||||||
|
def test_Youku(self):
|
||||||
|
filename = 'XNDgyMDQ2NTQw_part00.flv'
|
||||||
|
fd = FileDownloader(self.parameters)
|
||||||
|
fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE())
|
||||||
|
fd.download(['http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html'])
|
||||||
|
self.assertTrue(os.path.exists(filename))
|
||||||
|
md5_for_file = _file_md5(filename)
|
||||||
|
self.assertEqual(md5_for_file, 'ffe3f2e435663dc2d1eea34faeff5b5b')
|
||||||
|
|
||||||
|
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
if os.path.exists('BaW_jenozKc.mp4'):
|
if os.path.exists('BaW_jenozKc.mp4'):
|
||||||
os.remove('BaW_jenozKc.mp4')
|
os.remove('BaW_jenozKc.mp4')
|
||||||
if os.path.exists('x33vw9.mp4'):
|
if os.path.exists('x33vw9.mp4'):
|
||||||
os.remove('x33vw9.mp4')
|
os.remove('x33vw9.mp4')
|
||||||
if os.path.exists('aUehQsCQtM.flv'):
|
if os.path.exists('_aUehQsCQtM.flv'):
|
||||||
os.remove('aUehQsCQtM.flv')
|
os.remove('_aUehQsCQtM.flv')
|
||||||
if os.path.exists('5779306.m4v'):
|
if os.path.exists('5779306.m4v'):
|
||||||
os.remove('5779306.m4v')
|
os.remove('5779306.m4v')
|
||||||
if os.path.exists('939581.flv'):
|
if os.path.exists('939581.flv'):
|
||||||
os.remove('939581.flv')
|
os.remove('939581.flv')
|
||||||
# No file specified for Vimeo
|
# No file specified for Vimeo
|
||||||
if os.path.exists('n6FLbx6ZzMiu.mp3'):
|
if os.path.exists('62986583.mp3'):
|
||||||
os.remove('n6FLbx6ZzMiu.mp3')
|
os.remove('62986583.mp3')
|
||||||
if os.path.exists('PracticalUnix_intro-environment.mp4'):
|
if os.path.exists('PracticalUnix_intro-environment.mp4'):
|
||||||
os.remove('PracticalUnix_intro-environment.mp4')
|
os.remove('PracticalUnix_intro-environment.mp4')
|
||||||
# No file specified for CollegeHumor
|
# No file specified for CollegeHumor
|
||||||
if os.path.exists('1135332.flv'):
|
if os.path.exists('1135332.flv'):
|
||||||
os.remove('1135332.flv')
|
os.remove('1135332.flv')
|
||||||
|
if os.path.exists('XNDgyMDQ2NTQw_part00.flv'):
|
||||||
|
os.remove('XNDgyMDQ2NTQw_part00.flv')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
26
test/test_execution.py
Normal file
26
test/test_execution.py
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
try:
|
||||||
|
_DEV_NULL = subprocess.DEVNULL
|
||||||
|
except AttributeError:
|
||||||
|
_DEV_NULL = open(os.devnull, 'wb')
|
||||||
|
|
||||||
|
class TestExecution(unittest.TestCase):
|
||||||
|
def test_import(self):
|
||||||
|
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
|
||||||
|
|
||||||
|
def test_module_exec(self):
|
||||||
|
if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
|
||||||
|
subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
|
||||||
|
|
||||||
|
def test_main_exec(self):
|
||||||
|
subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
@ -1,13 +0,0 @@
|
|||||||
import unittest
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os.path
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
class TestImport(unittest.TestCase):
|
|
||||||
def test_import(self):
|
|
||||||
rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
||||||
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
@ -1,3 +1,5 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
# Various small unit tests
|
# Various small unit tests
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
@ -79,6 +81,11 @@ class TestUtil(unittest.TestCase):
|
|||||||
self.assertTrue(sanitize_filename('-', restricted=True) != '')
|
self.assertTrue(sanitize_filename('-', restricted=True) != '')
|
||||||
self.assertTrue(sanitize_filename(':', restricted=True) != '')
|
self.assertTrue(sanitize_filename(':', restricted=True) != '')
|
||||||
|
|
||||||
|
def test_sanitize_ids(self):
|
||||||
|
self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
|
||||||
|
self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
|
||||||
|
self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
|
||||||
|
|
||||||
def test_ordered_set(self):
|
def test_ordered_set(self):
|
||||||
self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
|
self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
|
||||||
self.assertEqual(orderedSet([]), [])
|
self.assertEqual(orderedSet([]), [])
|
||||||
|
22
test/test_youtube_playlist_ids.py
Normal file
22
test/test_youtube_playlist_ids.py
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
# Allow direct execution
|
||||||
|
import os
|
||||||
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
|
||||||
|
|
||||||
|
class TestYoutubePlaylistMatching(unittest.TestCase):
|
||||||
|
def test_playlist_matching(self):
|
||||||
|
assert YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
||||||
|
assert YoutubePlaylistIE().suitable(u'PL63F0C78739B09958')
|
||||||
|
assert not YoutubePlaylistIE().suitable(u'PLtS2H6bU1M')
|
||||||
|
|
||||||
|
def test_youtube_matching(self):
|
||||||
|
assert YoutubeIE().suitable(u'PLtS2H6bU1M')
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
@ -16,7 +16,7 @@
|
|||||||
"size": 5754305,
|
"size": 5754305,
|
||||||
"addIEs": ["Youtube"],
|
"addIEs": ["Youtube"],
|
||||||
"url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
|
"url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
|
||||||
"file": "aUehQsCQtM.flv"
|
"file": "_aUehQsCQtM.flv"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "BlipTV",
|
"name": "BlipTV",
|
||||||
@ -40,7 +40,7 @@
|
|||||||
"name": "Soundcloud",
|
"name": "Soundcloud",
|
||||||
"md5": "c1b9b9ea8bfd620b96b2628664576e1c",
|
"md5": "c1b9b9ea8bfd620b96b2628664576e1c",
|
||||||
"url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
|
"url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
|
||||||
"file": "n6FLbx6ZzMiu.mp3"
|
"file": "62986583.mp3"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "StanfordOpenClassroom",
|
"name": "StanfordOpenClassroom",
|
||||||
@ -54,10 +54,16 @@
|
|||||||
"url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody",
|
"url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody",
|
||||||
"file": ""
|
"file": ""
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "XNXX",
|
"name": "XNXX",
|
||||||
"md5": "c5c67df477eb0d9b058200351448ba4c",
|
"md5": "c5c67df477eb0d9b058200351448ba4c",
|
||||||
"url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
|
"url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
|
||||||
"file": "1135332.flv"
|
"file": "1135332.flv"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Youku",
|
||||||
|
"url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
|
||||||
|
"file": "XNDgyMDQ2NTQw_part00.flv",
|
||||||
|
"md5": "ffe3f2e435663dc2d1eea34faeff5b5b"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
@ -334,8 +334,11 @@ class FileDownloader(object):
|
|||||||
template_dict['epoch'] = int(time.time())
|
template_dict['epoch'] = int(time.time())
|
||||||
template_dict['autonumber'] = u'%05d' % self._num_downloads
|
template_dict['autonumber'] = u'%05d' % self._num_downloads
|
||||||
|
|
||||||
template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
|
sanitize = lambda k,v: sanitize_filename(
|
||||||
template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
|
u'NA' if v is None else compat_str(v),
|
||||||
|
restricted=self.params.get('restrictfilenames'),
|
||||||
|
is_id=(k==u'id'))
|
||||||
|
template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
|
||||||
|
|
||||||
filename = self.params['outtmpl'] % template_dict
|
filename = self.params['outtmpl'] % template_dict
|
||||||
return filename
|
return filename
|
||||||
|
@ -1671,7 +1671,7 @@ class YahooSearchIE(InfoExtractor):
|
|||||||
class YoutubePlaylistIE(InfoExtractor):
|
class YoutubePlaylistIE(InfoExtractor):
|
||||||
"""Information Extractor for YouTube playlists."""
|
"""Information Extractor for YouTube playlists."""
|
||||||
|
|
||||||
_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
|
_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
|
||||||
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
|
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
|
||||||
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
|
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
|
||||||
_MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
|
_MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
|
||||||
@ -2803,13 +2803,13 @@ class SoundcloudIE(InfoExtractor):
|
|||||||
def __init__(self, downloader=None):
|
def __init__(self, downloader=None):
|
||||||
InfoExtractor.__init__(self, downloader)
|
InfoExtractor.__init__(self, downloader)
|
||||||
|
|
||||||
def report_webpage(self, video_id):
|
def report_resolve(self, video_id):
|
||||||
"""Report information extraction."""
|
"""Report information extraction."""
|
||||||
self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
|
self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
|
||||||
|
|
||||||
def report_extraction(self, video_id):
|
def report_extraction(self, video_id):
|
||||||
"""Report information extraction."""
|
"""Report information extraction."""
|
||||||
self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
|
self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
@ -2818,65 +2818,47 @@ class SoundcloudIE(InfoExtractor):
|
|||||||
return
|
return
|
||||||
|
|
||||||
# extract uploader (which is in the url)
|
# extract uploader (which is in the url)
|
||||||
uploader = mobj.group(1).decode('utf-8')
|
uploader = mobj.group(1)
|
||||||
# extract simple title (uploader + slug of song title)
|
# extract simple title (uploader + slug of song title)
|
||||||
slug_title = mobj.group(2).decode('utf-8')
|
slug_title = mobj.group(2)
|
||||||
simple_title = uploader + u'-' + slug_title
|
simple_title = uploader + u'-' + slug_title
|
||||||
|
|
||||||
self.report_webpage('%s/%s' % (uploader, slug_title))
|
self.report_resolve('%s/%s' % (uploader, slug_title))
|
||||||
|
|
||||||
request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
|
url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
|
||||||
|
resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
|
||||||
|
request = compat_urllib_request.Request(resolv_url)
|
||||||
try:
|
try:
|
||||||
webpage = compat_urllib_request.urlopen(request).read()
|
info_json_bytes = compat_urllib_request.urlopen(request).read()
|
||||||
|
info_json = info_json_bytes.decode('utf-8')
|
||||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||||
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
|
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
|
||||||
return
|
return
|
||||||
|
|
||||||
|
info = json.loads(info_json)
|
||||||
|
video_id = info['id']
|
||||||
self.report_extraction('%s/%s' % (uploader, slug_title))
|
self.report_extraction('%s/%s' % (uploader, slug_title))
|
||||||
|
|
||||||
# extract uid and stream token that soundcloud hands out for access
|
streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
|
||||||
mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
|
request = compat_urllib_request.Request(streams_url)
|
||||||
if mobj:
|
try:
|
||||||
video_id = mobj.group(1)
|
stream_json_bytes = compat_urllib_request.urlopen(request).read()
|
||||||
stream_token = mobj.group(2)
|
stream_json = stream_json_bytes.decode('utf-8')
|
||||||
|
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||||
|
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
|
||||||
|
return
|
||||||
|
|
||||||
# extract unsimplified title
|
streams = json.loads(stream_json)
|
||||||
mobj = re.search('"title":"(.*?)",', webpage)
|
mediaURL = streams['http_mp3_128_url']
|
||||||
if mobj:
|
|
||||||
title = mobj.group(1).decode('utf-8')
|
|
||||||
else:
|
|
||||||
title = simple_title
|
|
||||||
|
|
||||||
# construct media url (with uid/token)
|
|
||||||
mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
|
|
||||||
mediaURL = mediaURL % (video_id, stream_token)
|
|
||||||
|
|
||||||
# description
|
|
||||||
description = u'No description available'
|
|
||||||
mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
|
|
||||||
if mobj:
|
|
||||||
description = mobj.group(1)
|
|
||||||
|
|
||||||
# upload date
|
|
||||||
upload_date = None
|
|
||||||
mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
|
|
||||||
if mobj:
|
|
||||||
try:
|
|
||||||
upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
|
|
||||||
except Exception as err:
|
|
||||||
self._downloader.to_stderr(compat_str(err))
|
|
||||||
|
|
||||||
# for soundcloud, a request to a cross domain is required for cookies
|
|
||||||
request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
|
|
||||||
|
|
||||||
return [{
|
return [{
|
||||||
'id': video_id.decode('utf-8'),
|
'id': info['id'],
|
||||||
'url': mediaURL,
|
'url': mediaURL,
|
||||||
'uploader': uploader.decode('utf-8'),
|
'uploader': info['user']['username'],
|
||||||
'upload_date': upload_date,
|
'upload_date': info['created_at'],
|
||||||
'title': title,
|
'title': info['title'],
|
||||||
'ext': u'mp3',
|
'ext': u'mp3',
|
||||||
'description': description.decode('utf-8')
|
'description': info['description'],
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
|
||||||
|
@ -305,7 +305,7 @@ def parseOpts():
|
|||||||
action='store_true', dest='autonumber',
|
action='store_true', dest='autonumber',
|
||||||
help='number downloaded files starting from 00000', default=False)
|
help='number downloaded files starting from 00000', default=False)
|
||||||
filesystem.add_option('-o', '--output',
|
filesystem.add_option('-o', '--output',
|
||||||
dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
|
dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
|
||||||
filesystem.add_option('--restrict-filenames',
|
filesystem.add_option('--restrict-filenames',
|
||||||
action='store_true', dest='restrictfilenames',
|
action='store_true', dest='restrictfilenames',
|
||||||
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
|
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
|
||||||
|
@ -317,9 +317,10 @@ def timeconvert(timestr):
|
|||||||
timestamp = email.utils.mktime_tz(timetuple)
|
timestamp = email.utils.mktime_tz(timetuple)
|
||||||
return timestamp
|
return timestamp
|
||||||
|
|
||||||
def sanitize_filename(s, restricted=False):
|
def sanitize_filename(s, restricted=False, is_id=False):
|
||||||
"""Sanitizes a string so it could be used as part of a filename.
|
"""Sanitizes a string so it could be used as part of a filename.
|
||||||
If restricted is set, use a stricter subset of allowed characters.
|
If restricted is set, use a stricter subset of allowed characters.
|
||||||
|
Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
|
||||||
"""
|
"""
|
||||||
def replace_insane(char):
|
def replace_insane(char):
|
||||||
if char == '?' or ord(char) < 32 or ord(char) == 127:
|
if char == '?' or ord(char) < 32 or ord(char) == 127:
|
||||||
@ -337,14 +338,15 @@ def sanitize_filename(s, restricted=False):
|
|||||||
return char
|
return char
|
||||||
|
|
||||||
result = u''.join(map(replace_insane, s))
|
result = u''.join(map(replace_insane, s))
|
||||||
while '__' in result:
|
if not is_id:
|
||||||
result = result.replace('__', '_')
|
while '__' in result:
|
||||||
result = result.strip('_')
|
result = result.replace('__', '_')
|
||||||
# Common case of "Foreign band name - English song title"
|
result = result.strip('_')
|
||||||
if restricted and result.startswith('-_'):
|
# Common case of "Foreign band name - English song title"
|
||||||
result = result[2:]
|
if restricted and result.startswith('-_'):
|
||||||
if not result:
|
result = result[2:]
|
||||||
result = '_'
|
if not result:
|
||||||
|
result = '_'
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def orderedSet(iterable):
|
def orderedSet(iterable):
|
||||||
@ -504,3 +506,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
|
|||||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||||
resp.msg = old_resp.msg
|
resp.msg = old_resp.msg
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
|
https_request = http_request
|
||||||
|
https_response = http_response
|
||||||
|
Loading…
Reference in New Issue
Block a user