mirror of
https://github.com/blackjack4494/yt-dlc.git
synced 2024-08-18 16:29:43 +02:00
Merge branch 'master' into extract_info_rewrite
This commit is contained in:
commit
14294236bf
14
README.md
14
README.md
@ -18,7 +18,7 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
--version print program version and exit
|
||||
-U, --update update this program to latest version
|
||||
-i, --ignore-errors continue on download errors
|
||||
-r, --rate-limit LIMIT download rate limit (e.g. 50k or 44.6m)
|
||||
-r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m)
|
||||
-R, --retries RETRIES number of retries (default is 10)
|
||||
--buffer-size SIZE size of download buffer (e.g. 1024 or 16k) (default
|
||||
is 1024)
|
||||
@ -97,10 +97,16 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
requested
|
||||
--max-quality FORMAT highest quality format to download
|
||||
-F, --list-formats list all available formats (currently youtube only)
|
||||
--write-srt write video closed captions to a .srt file
|
||||
--write-sub write subtitle file (currently youtube only)
|
||||
--only-sub downloads only the subtitles (no video)
|
||||
--all-subs downloads all the available subtitles of the video
|
||||
(currently youtube only)
|
||||
--srt-lang LANG language of the closed captions to download
|
||||
(optional) use IETF language tags like 'en'
|
||||
--list-subs lists all available subtitles for the video
|
||||
(currently youtube only)
|
||||
--sub-format LANG subtitle format [srt/sbv] (default=srt) (currently
|
||||
youtube only)
|
||||
--sub-lang LANG language of the subtitles to download (optional)
|
||||
use IETF language tags like 'en'
|
||||
|
||||
## Authentication Options:
|
||||
-u, --username USERNAME account username
|
||||
|
57
devscripts/gh-pages/update-feed.py
Executable file
57
devscripts/gh-pages/update-feed.py
Executable file
@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import datetime
|
||||
|
||||
import textwrap
|
||||
|
||||
import json
|
||||
|
||||
atom_template=textwrap.dedent("""\
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<atom:feed xmlns:atom="http://www.w3.org/2005/Atom">
|
||||
<atom:title>youtube-dl releases</atom:title>
|
||||
<atom:id>youtube-dl-updates-feed</atom:id>
|
||||
<atom:updated>@TIMESTAMP@</atom:updated>
|
||||
@ENTRIES@
|
||||
</atom:feed>""")
|
||||
|
||||
entry_template=textwrap.dedent("""
|
||||
<atom:entry>
|
||||
<atom:id>youtube-dl-@VERSION@</atom:id>
|
||||
<atom:title>New version @VERSION@</atom:title>
|
||||
<atom:link href="http://rg3.github.com/youtube-dl" />
|
||||
<atom:content type="xhtml">
|
||||
<div xmlns="http://www.w3.org/1999/xhtml">
|
||||
Downloads available at <a href="http://youtube-dl.org/downloads/@VERSION@/">http://youtube-dl.org/downloads/@VERSION@/</a>
|
||||
</div>
|
||||
</atom:content>
|
||||
<atom:author>
|
||||
<atom:name>The youtube-dl maintainers</atom:name>
|
||||
</atom:author>
|
||||
<atom:updated>@TIMESTAMP@</atom:updated>
|
||||
</atom:entry>
|
||||
""")
|
||||
|
||||
now = datetime.datetime.now()
|
||||
now_iso = now.isoformat()
|
||||
|
||||
atom_template = atom_template.replace('@TIMESTAMP@',now_iso)
|
||||
|
||||
entries=[]
|
||||
|
||||
versions_info = json.load(open('update/versions.json'))
|
||||
versions = list(versions_info['versions'].keys())
|
||||
versions.sort()
|
||||
|
||||
for v in versions:
|
||||
entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-'))
|
||||
entry = entry.replace('@VERSION@',v)
|
||||
entries.append(entry)
|
||||
|
||||
entries_str = textwrap.indent(''.join(entries), '\t')
|
||||
atom_template = atom_template.replace('@ENTRIES@', entries_str)
|
||||
|
||||
with open('update/releases.atom','w',encoding='utf-8') as atom_file:
|
||||
atom_file.write(atom_template)
|
||||
|
||||
|
@ -69,6 +69,7 @@ ROOT=$(pwd)
|
||||
ORIGIN_URL=$(git config --get remote.origin.url)
|
||||
cd build/gh-pages
|
||||
"$ROOT/devscripts/gh-pages/add-version.py" $version
|
||||
"$ROOT/devscripts/gh-pages/update-feed.py"
|
||||
"$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
|
||||
"$ROOT/devscripts/gh-pages/generate-download.py"
|
||||
"$ROOT/devscripts/gh-pages/update-copyright.py"
|
||||
|
@ -20,6 +20,8 @@ from youtube_dl.utils import *
|
||||
DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
|
||||
PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
|
||||
|
||||
RETRIES = 3
|
||||
|
||||
# General configuration (from __init__, not very elegant...)
|
||||
jar = compat_cookiejar.CookieJar()
|
||||
cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
|
||||
@ -79,9 +81,8 @@ def generator(test_case):
|
||||
params.update(test_case.get('params', {}))
|
||||
|
||||
fd = FileDownloader(params)
|
||||
fd.add_info_extractor(ie())
|
||||
for ien in test_case.get('add_ie', []):
|
||||
fd.add_info_extractor(getattr(youtube_dl.InfoExtractors, ien + 'IE')())
|
||||
for ie in youtube_dl.InfoExtractors.gen_extractors():
|
||||
fd.add_info_extractor(ie)
|
||||
finished_hook_called = set()
|
||||
def _hook(status):
|
||||
if status['status'] == 'finished':
|
||||
@ -93,8 +94,20 @@ def generator(test_case):
|
||||
_try_rm(tc['file'])
|
||||
_try_rm(tc['file'] + '.part')
|
||||
_try_rm(tc['file'] + '.info.json')
|
||||
try:
|
||||
for retry in range(1, RETRIES + 1):
|
||||
try:
|
||||
fd.download([test_case['url']])
|
||||
except (DownloadError, ExtractorError) as err:
|
||||
if retry == RETRIES: raise
|
||||
|
||||
# Check if the exception is not a network related one
|
||||
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
|
||||
raise
|
||||
|
||||
print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
|
||||
else:
|
||||
break
|
||||
|
||||
for tc in test_cases:
|
||||
if not test_case.get('params', {}).get('skip_download', False):
|
||||
|
@ -308,5 +308,25 @@
|
||||
"info_dict": {
|
||||
"title": "Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "LiveLeak",
|
||||
"md5": "0813c2430bea7a46bf13acf3406992f4",
|
||||
"url": "http://www.liveleak.com/view?i=757_1364311680",
|
||||
"file": "757_1364311680.mp4",
|
||||
"info_dict": {
|
||||
"title": "Most unlucky car accident",
|
||||
"description": "extremely bad day for this guy..!",
|
||||
"uploader": "ljfriel2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "WorldStarHipHop",
|
||||
"url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
|
||||
"file": "wshh6a7q1ny0G34ZwuIO.mp4",
|
||||
"md5": "9d04de741161603bf7071bbf4e883186",
|
||||
"info_dict": {
|
||||
"title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick! "
|
||||
}
|
||||
}
|
||||
]
|
||||
|
BIN
youtube-dl
BIN
youtube-dl
Binary file not shown.
@ -231,11 +231,21 @@ class FileDownloader(object):
|
||||
self.to_stderr(message)
|
||||
if self.params.get('verbose'):
|
||||
if tb is None:
|
||||
if sys.exc_info()[0]: # if .trouble has been called from an except block
|
||||
tb = u''
|
||||
if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
|
||||
tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
|
||||
tb += compat_str(traceback.format_exc())
|
||||
else:
|
||||
tb_data = traceback.format_list(traceback.extract_stack())
|
||||
tb = u''.join(tb_data)
|
||||
self.to_stderr(tb)
|
||||
if not self.params.get('ignoreerrors', False):
|
||||
raise DownloadError(message)
|
||||
if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
|
||||
exc_info = sys.exc_info()[1].exc_info
|
||||
else:
|
||||
exc_info = sys.exc_info()
|
||||
raise DownloadError(message, exc_info)
|
||||
self._download_retcode = 1
|
||||
|
||||
def report_warning(self, message):
|
||||
@ -250,6 +260,18 @@ class FileDownloader(object):
|
||||
warning_message=u'%s %s' % (_msg_header,message)
|
||||
self.to_stderr(warning_message)
|
||||
|
||||
def report_error(self, message, tb=None):
|
||||
'''
|
||||
Do the same as trouble, but prefixes the message with 'ERROR:', colored
|
||||
in red if stderr is a tty file.
|
||||
'''
|
||||
if sys.stderr.isatty():
|
||||
_msg_header = u'\033[0;31mERROR:\033[0m'
|
||||
else:
|
||||
_msg_header = u'ERROR:'
|
||||
error_message = u'%s %s' % (_msg_header, message)
|
||||
self.trouble(error_message, tb)
|
||||
|
||||
def slow_down(self, start_time, byte_counter):
|
||||
"""Sleep if the download speed is over the rate limit."""
|
||||
rate_limit = self.params.get('ratelimit', None)
|
||||
@ -281,7 +303,7 @@ class FileDownloader(object):
|
||||
return
|
||||
os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
|
||||
except (IOError, OSError) as err:
|
||||
self.trouble(u'ERROR: unable to rename file')
|
||||
self.report_error(u'unable to rename file')
|
||||
|
||||
def try_utime(self, filename, last_modified_hdr):
|
||||
"""Try to set the last-modified time of the given file."""
|
||||
@ -519,7 +541,7 @@ class FileDownloader(object):
|
||||
if dn != '' and not os.path.exists(dn): # dn is already encoded
|
||||
os.makedirs(dn)
|
||||
except (OSError, IOError) as err:
|
||||
self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
|
||||
self.report_error(u'unable to create directory ' + compat_str(err))
|
||||
return
|
||||
|
||||
if self.params.get('writedescription', False):
|
||||
@ -529,7 +551,7 @@ class FileDownloader(object):
|
||||
with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
|
||||
descfile.write(info_dict['description'])
|
||||
except (OSError, IOError):
|
||||
self.trouble(u'ERROR: Cannot write description file ' + descfn)
|
||||
self.report_error(u'Cannot write description file ' + descfn)
|
||||
return
|
||||
|
||||
if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
|
||||
@ -538,13 +560,16 @@ class FileDownloader(object):
|
||||
subtitle = info_dict['subtitles'][0]
|
||||
(sub_error, sub_lang, sub) = subtitle
|
||||
sub_format = self.params.get('subtitlesformat')
|
||||
if sub_error:
|
||||
self.report_warning("Some error while getting the subtitles")
|
||||
else:
|
||||
try:
|
||||
sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
|
||||
self.report_writesubtitles(sub_filename)
|
||||
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
|
||||
subfile.write(sub)
|
||||
except (OSError, IOError):
|
||||
self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
|
||||
self.report_error(u'Cannot write subtitles file ' + descfn)
|
||||
return
|
||||
if self.params.get('onlysubtitles', False):
|
||||
return
|
||||
@ -554,6 +579,9 @@ class FileDownloader(object):
|
||||
sub_format = self.params.get('subtitlesformat')
|
||||
for subtitle in subtitles:
|
||||
(sub_error, sub_lang, sub) = subtitle
|
||||
if sub_error:
|
||||
self.report_warning("Some error while getting the subtitles")
|
||||
else:
|
||||
try:
|
||||
sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
|
||||
self.report_writesubtitles(sub_filename)
|
||||
@ -572,7 +600,7 @@ class FileDownloader(object):
|
||||
json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
|
||||
write_json_file(json_info_dict, encodeFilename(infofn))
|
||||
except (OSError, IOError):
|
||||
self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
|
||||
self.report_error(u'Cannot write metadata to JSON file ' + infofn)
|
||||
return
|
||||
|
||||
if not self.params.get('skip_download', False):
|
||||
@ -584,17 +612,17 @@ class FileDownloader(object):
|
||||
except (OSError, IOError) as err:
|
||||
raise UnavailableVideoError()
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self.trouble(u'ERROR: unable to download video data: %s' % str(err))
|
||||
self.report_error(u'unable to download video data: %s' % str(err))
|
||||
return
|
||||
except (ContentTooShortError, ) as err:
|
||||
self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
|
||||
self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
|
||||
return
|
||||
|
||||
if success:
|
||||
try:
|
||||
self.post_process(filename, info_dict)
|
||||
except (PostProcessingError) as err:
|
||||
self.trouble(u'ERROR: postprocessing: %s' % str(err))
|
||||
self.report_error(u'postprocessing: %s' % str(err))
|
||||
return
|
||||
|
||||
def download(self, url_list):
|
||||
@ -611,6 +639,9 @@ class FileDownloader(object):
|
||||
self.process_info(video)
|
||||
except UnavailableVideoError:
|
||||
self.trouble(u'\nERROR: unable to download video')
|
||||
except MaxDownloadsReached:
|
||||
self.to_screen(u'[info] Maximum number of downloaded files reached.')
|
||||
raise
|
||||
|
||||
return self._download_retcode
|
||||
|
||||
@ -645,7 +676,7 @@ class FileDownloader(object):
|
||||
try:
|
||||
subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
|
||||
except (OSError, IOError):
|
||||
self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
|
||||
self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
|
||||
return False
|
||||
|
||||
# Download using rtmpdump. rtmpdump returns exit code 2 when
|
||||
@ -690,7 +721,8 @@ class FileDownloader(object):
|
||||
})
|
||||
return True
|
||||
else:
|
||||
self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
|
||||
self.to_stderr(u"\n")
|
||||
self.report_error(u'rtmpdump exited with code %d' % retval)
|
||||
return False
|
||||
|
||||
def _do_download(self, filename, info_dict):
|
||||
@ -790,7 +822,7 @@ class FileDownloader(object):
|
||||
self.report_retry(count, retries)
|
||||
|
||||
if count > retries:
|
||||
self.trouble(u'ERROR: giving up after %s retries' % retries)
|
||||
self.report_error(u'giving up after %s retries' % retries)
|
||||
return False
|
||||
|
||||
data_len = data.info().get('Content-length', None)
|
||||
@ -826,12 +858,13 @@ class FileDownloader(object):
|
||||
filename = self.undo_temp_name(tmpfilename)
|
||||
self.report_destination(filename)
|
||||
except (OSError, IOError) as err:
|
||||
self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
|
||||
self.report_error(u'unable to open for writing: %s' % str(err))
|
||||
return False
|
||||
try:
|
||||
stream.write(data_block)
|
||||
except (IOError, OSError) as err:
|
||||
self.trouble(u'\nERROR: unable to write data: %s' % str(err))
|
||||
self.to_stderr(u"\n")
|
||||
self.report_error(u'unable to write data: %s' % str(err))
|
||||
return False
|
||||
if not self.params.get('noresizebuffer', False):
|
||||
block_size = self.best_block_size(after - before, len(data_block))
|
||||
@ -857,7 +890,8 @@ class FileDownloader(object):
|
||||
self.slow_down(start, byte_counter - resume_len)
|
||||
|
||||
if stream is None:
|
||||
self.trouble(u'\nERROR: Did not get any data blocks')
|
||||
self.to_stderr(u"\n")
|
||||
self.report_error(u'Did not get any data blocks')
|
||||
return False
|
||||
stream.close()
|
||||
self.report_finish()
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -311,7 +311,7 @@ def clean_html(html):
|
||||
html = re.sub('<.*?>', '', html)
|
||||
# Replace html entities
|
||||
html = unescapeHTML(html)
|
||||
return html
|
||||
return html.strip()
|
||||
|
||||
|
||||
def sanitize_open(filename, open_mode):
|
||||
@ -329,7 +329,7 @@ def sanitize_open(filename, open_mode):
|
||||
if sys.platform == 'win32':
|
||||
import msvcrt
|
||||
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
|
||||
return (sys.stdout, filename)
|
||||
return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
|
||||
stream = open(encodeFilename(filename), open_mode)
|
||||
return (stream, filename)
|
||||
except (IOError, OSError) as err:
|
||||
@ -435,6 +435,7 @@ class ExtractorError(Exception):
|
||||
""" tb, if given, is the original traceback (so that it can be printed out). """
|
||||
super(ExtractorError, self).__init__(msg)
|
||||
self.traceback = tb
|
||||
self.exc_info = sys.exc_info() # preserve original exception
|
||||
|
||||
def format_traceback(self):
|
||||
if self.traceback is None:
|
||||
@ -449,7 +450,10 @@ class DownloadError(Exception):
|
||||
configured to continue on errors. They will contain the appropriate
|
||||
error message.
|
||||
"""
|
||||
pass
|
||||
def __init__(self, msg, exc_info=None):
|
||||
""" exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
|
||||
super(DownloadError, self).__init__(msg)
|
||||
self.exc_info = exc_info
|
||||
|
||||
|
||||
class SameFileError(Exception):
|
||||
|
@ -1,2 +1,2 @@
|
||||
|
||||
__version__ = '2013.02.25'
|
||||
__version__ = '2013.04.03'
|
||||
|
Loading…
Reference in New Issue
Block a user