mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-09-05 02:19:36 +02:00
Option to choose different downloader for different protocols
* Renamed `--external-downloader-args` to `--downloader-args` * Added `native` as an option for the downloader * Use similar syntax to `--downloader-args` etc. Eg: `--downloader dash:native --downloader aria2c` * Deprecated `--hls-prefer-native` and `--hls-prefer-ffmpeg` since the same can now be done with `--downloader "m3u8:native"` and `m3u8:ffmpeg` respectively * Split `frag_urls` protocol into `m3u8_frag_urls` and `dash_frag_urls` * Standardize shortening of protocol names with `downloader.shorten_protocol_name`
This commit is contained in:
parent
d818eb7473
commit
52a8a1e1b9
23
README.md
23
README.md
@ -337,10 +337,6 @@ ## Download Options:
|
|||||||
--playlist-random Download playlist videos in random order
|
--playlist-random Download playlist videos in random order
|
||||||
--xattr-set-filesize Set file xattribute ytdl.filesize with
|
--xattr-set-filesize Set file xattribute ytdl.filesize with
|
||||||
expected file size
|
expected file size
|
||||||
--hls-prefer-native Use the native HLS downloader instead of
|
|
||||||
ffmpeg
|
|
||||||
--hls-prefer-ffmpeg Use ffmpeg instead of the native HLS
|
|
||||||
downloader
|
|
||||||
--hls-use-mpegts Use the mpegts container for HLS videos;
|
--hls-use-mpegts Use the mpegts container for HLS videos;
|
||||||
allowing some players to play the video
|
allowing some players to play the video
|
||||||
while downloading, and reducing the chance
|
while downloading, and reducing the chance
|
||||||
@ -350,10 +346,19 @@ ## Download Options:
|
|||||||
--no-hls-use-mpegts Do not use the mpegts container for HLS
|
--no-hls-use-mpegts Do not use the mpegts container for HLS
|
||||||
videos. This is default when not
|
videos. This is default when not
|
||||||
downloading live streams
|
downloading live streams
|
||||||
--external-downloader NAME Name or path of the external downloader to
|
--downloader [PROTO:]NAME Name or path of the external downloader to
|
||||||
use. Currently supports aria2c, avconv,
|
use (optionally) prefixed by the protocols
|
||||||
axel, curl, ffmpeg, httpie, wget
|
(http, ftp, m3u8, dash, rstp, rtmp, mms) to
|
||||||
(Recommended: aria2c)
|
use it for. Currently supports native,
|
||||||
|
aria2c, avconv, axel, curl, ffmpeg, httpie,
|
||||||
|
wget (Recommended: aria2c). You can use
|
||||||
|
this option multiple times to set different
|
||||||
|
downloaders for different protocols. For
|
||||||
|
example, --downloader aria2c --downloader
|
||||||
|
"dash,m3u8:native" will use aria2c for
|
||||||
|
http/ftp downloads, and the native
|
||||||
|
downloader for dash/m3u8 downloads
|
||||||
|
(Alias: --external-downloader)
|
||||||
--downloader-args NAME:ARGS Give these arguments to the external
|
--downloader-args NAME:ARGS Give these arguments to the external
|
||||||
downloader. Specify the downloader name and
|
downloader. Specify the downloader name and
|
||||||
the arguments separated by a colon ":". You
|
the arguments separated by a colon ":". You
|
||||||
@ -1244,6 +1249,8 @@ # DEPRECATED OPTIONS
|
|||||||
--metadata-from-title FORMAT --parse-metadata "%(title)s:FORMAT"
|
--metadata-from-title FORMAT --parse-metadata "%(title)s:FORMAT"
|
||||||
--prefer-avconv avconv is no longer officially supported (Alias: --no-prefer-ffmpeg)
|
--prefer-avconv avconv is no longer officially supported (Alias: --no-prefer-ffmpeg)
|
||||||
--prefer-ffmpeg Default (Alias: --no-prefer-avconv)
|
--prefer-ffmpeg Default (Alias: --no-prefer-avconv)
|
||||||
|
--hls-prefer-native --downloader "m3u8:native"
|
||||||
|
--hls-prefer-ffmpeg --downloader "m3u8:ffmpeg"
|
||||||
--avconv-location avconv is no longer officially supported
|
--avconv-location avconv is no longer officially supported
|
||||||
-C, --call-home Not implemented
|
-C, --call-home Not implemented
|
||||||
--no-call-home Default
|
--no-call-home Default
|
||||||
|
@ -111,9 +111,17 @@
|
|||||||
process_communicate_or_kill,
|
process_communicate_or_kill,
|
||||||
)
|
)
|
||||||
from .cache import Cache
|
from .cache import Cache
|
||||||
from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER, _PLUGIN_CLASSES
|
from .extractor import (
|
||||||
|
gen_extractor_classes,
|
||||||
|
get_info_extractor,
|
||||||
|
_LAZY_LOADER,
|
||||||
|
_PLUGIN_CLASSES
|
||||||
|
)
|
||||||
from .extractor.openload import PhantomJSwrapper
|
from .extractor.openload import PhantomJSwrapper
|
||||||
from .downloader import get_suitable_downloader
|
from .downloader import (
|
||||||
|
get_suitable_downloader,
|
||||||
|
shorten_protocol_name
|
||||||
|
)
|
||||||
from .downloader.rtmp import rtmpdump_version
|
from .downloader.rtmp import rtmpdump_version
|
||||||
from .postprocessor import (
|
from .postprocessor import (
|
||||||
FFmpegFixupM3u8PP,
|
FFmpegFixupM3u8PP,
|
||||||
@ -359,9 +367,13 @@ class YoutubeDL(object):
|
|||||||
geo_bypass_country
|
geo_bypass_country
|
||||||
|
|
||||||
The following options determine which downloader is picked:
|
The following options determine which downloader is picked:
|
||||||
external_downloader: Executable of the external downloader to call.
|
external_downloader: A dictionary of protocol keys and the executable of the
|
||||||
None or unset for standard (built-in) downloader.
|
external downloader to use for it. The allowed protocols
|
||||||
hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
|
are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
|
||||||
|
Set the value to 'native' to use the native downloader
|
||||||
|
hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
|
||||||
|
or {'m3u8': 'ffmpeg'} instead.
|
||||||
|
Use the native HLS downloader instead of ffmpeg/avconv
|
||||||
if True, otherwise use ffmpeg/avconv if False, otherwise
|
if True, otherwise use ffmpeg/avconv if False, otherwise
|
||||||
use downloader suggested by extractor if None.
|
use downloader suggested by extractor if None.
|
||||||
|
|
||||||
@ -2776,7 +2788,7 @@ def list_formats(self, info_dict):
|
|||||||
'|',
|
'|',
|
||||||
format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
|
format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
|
||||||
format_field(f, 'tbr', '%4dk'),
|
format_field(f, 'tbr', '%4dk'),
|
||||||
f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''),
|
shorten_protocol_name(f.get('protocol', '').replace("native", "n")),
|
||||||
'|',
|
'|',
|
||||||
format_field(f, 'vcodec', default='unknown').replace('none', ''),
|
format_field(f, 'vcodec', default='unknown').replace('none', ''),
|
||||||
format_field(f, 'vbr', '%4dk'),
|
format_field(f, 'vbr', '%4dk'),
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from ..compat import compat_str
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_protocol,
|
determine_protocol,
|
||||||
)
|
)
|
||||||
@ -42,6 +43,23 @@ def _get_real_downloader(info_dict, protocol=None, *args, **kwargs):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def shorten_protocol_name(proto, simplify=False):
|
||||||
|
short_protocol_names = {
|
||||||
|
'm3u8_native': 'm3u8_n',
|
||||||
|
'http_dash_segments': 'dash',
|
||||||
|
'niconico_dmc': 'dmc',
|
||||||
|
}
|
||||||
|
if simplify:
|
||||||
|
short_protocol_names.update({
|
||||||
|
'https': 'http',
|
||||||
|
'ftps': 'ftp',
|
||||||
|
'm3u8_native': 'm3u8',
|
||||||
|
'm3u8_frag_urls': 'm3u8',
|
||||||
|
'dash_frag_urls': 'dash',
|
||||||
|
})
|
||||||
|
return short_protocol_names.get(proto, proto)
|
||||||
|
|
||||||
|
|
||||||
def get_suitable_downloader(info_dict, params={}, default=HttpFD):
|
def get_suitable_downloader(info_dict, params={}, default=HttpFD):
|
||||||
"""Get the downloader class that can handle the info dict."""
|
"""Get the downloader class that can handle the info dict."""
|
||||||
protocol = determine_protocol(info_dict)
|
protocol = determine_protocol(info_dict)
|
||||||
@ -50,8 +68,14 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
|
|||||||
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
|
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
|
||||||
# return FFmpegFD
|
# return FFmpegFD
|
||||||
|
|
||||||
external_downloader = params.get('external_downloader')
|
downloaders = params.get('external_downloader')
|
||||||
if external_downloader is not None:
|
external_downloader = (
|
||||||
|
downloaders if isinstance(downloaders, compat_str)
|
||||||
|
else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default')))
|
||||||
|
if external_downloader and external_downloader.lower() == 'native':
|
||||||
|
external_downloader = 'native'
|
||||||
|
|
||||||
|
if external_downloader not in (None, 'native'):
|
||||||
ed = get_external_downloader(external_downloader)
|
ed = get_external_downloader(external_downloader)
|
||||||
if ed.can_download(info_dict, external_downloader):
|
if ed.can_download(info_dict, external_downloader):
|
||||||
return ed
|
return ed
|
||||||
@ -59,6 +83,8 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
|
|||||||
if protocol.startswith('m3u8'):
|
if protocol.startswith('m3u8'):
|
||||||
if info_dict.get('is_live'):
|
if info_dict.get('is_live'):
|
||||||
return FFmpegFD
|
return FFmpegFD
|
||||||
|
elif external_downloader == 'native':
|
||||||
|
return HlsFD
|
||||||
elif _get_real_downloader(info_dict, 'frag_urls', params, None):
|
elif _get_real_downloader(info_dict, 'frag_urls', params, None):
|
||||||
return HlsFD
|
return HlsFD
|
||||||
elif params.get('hls_prefer_native') is True:
|
elif params.get('hls_prefer_native') is True:
|
||||||
@ -70,6 +96,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
|
|||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
'get_suitable_downloader',
|
|
||||||
'FileDownloader',
|
'FileDownloader',
|
||||||
|
'get_suitable_downloader',
|
||||||
|
'shorten_protocol_name',
|
||||||
]
|
]
|
||||||
|
@ -20,7 +20,7 @@
|
|||||||
class DashSegmentsFD(FragmentFD):
|
class DashSegmentsFD(FragmentFD):
|
||||||
"""
|
"""
|
||||||
Download segments in a DASH manifest. External downloaders can take over
|
Download segments in a DASH manifest. External downloaders can take over
|
||||||
the fragment downloads by supporting the 'frag_urls' protocol
|
the fragment downloads by supporting the 'dash_frag_urls' protocol
|
||||||
"""
|
"""
|
||||||
|
|
||||||
FD_NAME = 'dashsegments'
|
FD_NAME = 'dashsegments'
|
||||||
@ -30,7 +30,7 @@ def real_download(self, filename, info_dict):
|
|||||||
fragments = info_dict['fragments'][:1] if self.params.get(
|
fragments = info_dict['fragments'][:1] if self.params.get(
|
||||||
'test', False) else info_dict['fragments']
|
'test', False) else info_dict['fragments']
|
||||||
|
|
||||||
real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
|
real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None)
|
||||||
|
|
||||||
ctx = {
|
ctx = {
|
||||||
'filename': filename,
|
'filename': filename,
|
||||||
|
@ -81,11 +81,15 @@ def get_basename(cls):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def exe(self):
|
def exe(self):
|
||||||
return self.params.get('external_downloader')
|
return self.get_basename()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def available(cls, path=None):
|
def available(cls, path=None):
|
||||||
return check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
|
path = check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
|
||||||
|
if path:
|
||||||
|
cls.exe = path
|
||||||
|
return path
|
||||||
|
return False
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def supports(cls, info_dict):
|
def supports(cls, info_dict):
|
||||||
@ -259,7 +263,7 @@ def _make_cmd(self, tmpfilename, info_dict):
|
|||||||
|
|
||||||
class Aria2cFD(ExternalFD):
|
class Aria2cFD(ExternalFD):
|
||||||
AVAILABLE_OPT = '-v'
|
AVAILABLE_OPT = '-v'
|
||||||
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'frag_urls')
|
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls')
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def supports_manifest(manifest):
|
def supports_manifest(manifest):
|
||||||
@ -310,9 +314,11 @@ def _make_cmd(self, tmpfilename, info_dict):
|
|||||||
|
|
||||||
|
|
||||||
class HttpieFD(ExternalFD):
|
class HttpieFD(ExternalFD):
|
||||||
|
AVAILABLE_OPT = '--version'
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def available(cls, path=None):
|
def available(cls, path=None):
|
||||||
return check_executable(path or 'http', ['--version'])
|
return ExternalFD.available(cls, path or 'http')
|
||||||
|
|
||||||
def _make_cmd(self, tmpfilename, info_dict):
|
def _make_cmd(self, tmpfilename, info_dict):
|
||||||
cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
|
cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
|
||||||
@ -327,7 +333,8 @@ class FFmpegFD(ExternalFD):
|
|||||||
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
|
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def available(cls, path=None): # path is ignored for ffmpeg
|
def available(cls, path=None):
|
||||||
|
# TODO: Fix path for ffmpeg
|
||||||
return FFmpegPostProcessor().available
|
return FFmpegPostProcessor().available
|
||||||
|
|
||||||
def _call_downloader(self, tmpfilename, info_dict):
|
def _call_downloader(self, tmpfilename, info_dict):
|
||||||
@ -484,4 +491,4 @@ def get_external_downloader(external_downloader):
|
|||||||
downloader . """
|
downloader . """
|
||||||
# Drop .exe extension on Windows
|
# Drop .exe extension on Windows
|
||||||
bn = os.path.splitext(os.path.basename(external_downloader))[0]
|
bn = os.path.splitext(os.path.basename(external_downloader))[0]
|
||||||
return _BY_NAME[bn]
|
return _BY_NAME.get(bn)
|
||||||
|
@ -32,7 +32,7 @@
|
|||||||
class HlsFD(FragmentFD):
|
class HlsFD(FragmentFD):
|
||||||
"""
|
"""
|
||||||
Download segments in a m3u8 manifest. External downloaders can take over
|
Download segments in a m3u8 manifest. External downloaders can take over
|
||||||
the fragment downloads by supporting the 'frag_urls' protocol and
|
the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
|
||||||
re-defining 'supports_manifest' function
|
re-defining 'supports_manifest' function
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@ -95,7 +95,7 @@ def real_download(self, filename, info_dict):
|
|||||||
# fd.add_progress_hook(ph)
|
# fd.add_progress_hook(ph)
|
||||||
return fd.real_download(filename, info_dict)
|
return fd.real_download(filename, info_dict)
|
||||||
|
|
||||||
real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
|
real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
|
||||||
if real_downloader and not real_downloader.supports_manifest(s):
|
if real_downloader and not real_downloader.supports_manifest(s):
|
||||||
real_downloader = None
|
real_downloader = None
|
||||||
if real_downloader:
|
if real_downloader:
|
||||||
|
@ -639,11 +639,11 @@ def _dict_from_multiple_values_options_callback(
|
|||||||
downloader.add_option(
|
downloader.add_option(
|
||||||
'--hls-prefer-native',
|
'--hls-prefer-native',
|
||||||
dest='hls_prefer_native', action='store_true', default=None,
|
dest='hls_prefer_native', action='store_true', default=None,
|
||||||
help='Use the native HLS downloader instead of ffmpeg')
|
help=optparse.SUPPRESS_HELP)
|
||||||
downloader.add_option(
|
downloader.add_option(
|
||||||
'--hls-prefer-ffmpeg',
|
'--hls-prefer-ffmpeg',
|
||||||
dest='hls_prefer_native', action='store_false', default=None,
|
dest='hls_prefer_native', action='store_false', default=None,
|
||||||
help='Use ffmpeg instead of the native HLS downloader')
|
help=optparse.SUPPRESS_HELP)
|
||||||
downloader.add_option(
|
downloader.add_option(
|
||||||
'--hls-use-mpegts',
|
'--hls-use-mpegts',
|
||||||
dest='hls_use_mpegts', action='store_true', default=None,
|
dest='hls_use_mpegts', action='store_true', default=None,
|
||||||
@ -659,11 +659,20 @@ def _dict_from_multiple_values_options_callback(
|
|||||||
'Do not use the mpegts container for HLS videos. '
|
'Do not use the mpegts container for HLS videos. '
|
||||||
'This is default when not downloading live streams'))
|
'This is default when not downloading live streams'))
|
||||||
downloader.add_option(
|
downloader.add_option(
|
||||||
'--external-downloader',
|
'--downloader', '--external-downloader',
|
||||||
dest='external_downloader', metavar='NAME',
|
dest='external_downloader', metavar='[PROTO:]NAME', default={}, type='str',
|
||||||
|
action='callback', callback=_dict_from_multiple_values_options_callback,
|
||||||
|
callback_kwargs={
|
||||||
|
'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
|
||||||
|
'default_key': 'default', 'process': lambda x: x.strip()},
|
||||||
help=(
|
help=(
|
||||||
'Name or path of the external downloader to use. '
|
'Name or path of the external downloader to use (optionally) prefixed by '
|
||||||
'Currently supports %s (Recommended: aria2c)' % ', '.join(list_external_downloaders())))
|
'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
|
||||||
|
'Currently supports native, %s (Recommended: aria2c). '
|
||||||
|
'You can use this option multiple times to set different downloaders for different protocols. '
|
||||||
|
'For example, --downloader aria2c --downloader "dash,m3u8:native" will use '
|
||||||
|
'aria2c for http/ftp downloads, and the native downloader for dash/m3u8 downloads '
|
||||||
|
'(Alias: --external-downloader)' % ', '.join(list_external_downloaders())))
|
||||||
downloader.add_option(
|
downloader.add_option(
|
||||||
'--downloader-args', '--external-downloader-args',
|
'--downloader-args', '--external-downloader-args',
|
||||||
metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',
|
metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',
|
||||||
|
Loading…
Reference in New Issue
Block a user