mirror of
https://github.com/yt-dlp/yt-dlp.git
synced 2024-09-05 18:39:37 +02:00
[cleanup] Refactor some code
This commit is contained in:
parent
d74a58a186
commit
dbf5416a20
@ -126,6 +126,7 @@
|
|||||||
)
|
)
|
||||||
from .extractor.openload import PhantomJSwrapper
|
from .extractor.openload import PhantomJSwrapper
|
||||||
from .downloader import (
|
from .downloader import (
|
||||||
|
FFmpegFD,
|
||||||
get_suitable_downloader,
|
get_suitable_downloader,
|
||||||
shorten_protocol_name
|
shorten_protocol_name
|
||||||
)
|
)
|
||||||
@ -2690,18 +2691,13 @@ def correct_ext(filename):
|
|||||||
info_dict['__real_download'] = False
|
info_dict['__real_download'] = False
|
||||||
|
|
||||||
_protocols = set(determine_protocol(f) for f in requested_formats)
|
_protocols = set(determine_protocol(f) for f in requested_formats)
|
||||||
if len(_protocols) == 1:
|
if len(_protocols) == 1: # All requested formats have same protocol
|
||||||
info_dict['protocol'] = _protocols.pop()
|
info_dict['protocol'] = _protocols.pop()
|
||||||
directly_mergable = (
|
directly_mergable = FFmpegFD.can_merge_formats(info_dict)
|
||||||
'no-direct-merge' not in self.params.get('compat_opts', [])
|
if dl_filename is not None:
|
||||||
and info_dict.get('protocol') is not None # All requested formats have same protocol
|
pass
|
||||||
and not self.params.get('allow_unplayable_formats')
|
elif (directly_mergable and get_suitable_downloader(info_dict, self.params) == FFmpegFD):
|
||||||
and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD')
|
info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
|
||||||
if directly_mergable:
|
|
||||||
info_dict['url'] = requested_formats[0]['url']
|
|
||||||
# Treat it as a single download
|
|
||||||
dl_filename = existing_file(full_filename, temp_filename)
|
|
||||||
if dl_filename is None:
|
|
||||||
success, real_download = self.dl(temp_filename, info_dict)
|
success, real_download = self.dl(temp_filename, info_dict)
|
||||||
info_dict['__real_download'] = real_download
|
info_dict['__real_download'] = real_download
|
||||||
else:
|
else:
|
||||||
@ -2717,14 +2713,11 @@ def correct_ext(filename):
|
|||||||
'You have requested merging of multiple formats but ffmpeg is not installed. '
|
'You have requested merging of multiple formats but ffmpeg is not installed. '
|
||||||
'The formats won\'t be merged.')
|
'The formats won\'t be merged.')
|
||||||
|
|
||||||
if dl_filename is None:
|
|
||||||
for f in requested_formats:
|
for f in requested_formats:
|
||||||
new_info = dict(info_dict)
|
new_info = dict(info_dict)
|
||||||
del new_info['requested_formats']
|
del new_info['requested_formats']
|
||||||
new_info.update(f)
|
new_info.update(f)
|
||||||
fname = prepend_extension(
|
fname = prepend_extension(temp_filename, 'f%s' % f['format_id'], new_info['ext'])
|
||||||
self.prepare_filename(new_info, 'temp'),
|
|
||||||
'f%s' % f['format_id'], new_info['ext'])
|
|
||||||
if not self._ensure_dir_exists(fname):
|
if not self._ensure_dir_exists(fname):
|
||||||
return
|
return
|
||||||
downloaded.append(fname)
|
downloaded.append(fname)
|
||||||
|
@ -3,17 +3,19 @@
|
|||||||
from ..compat import compat_str
|
from ..compat import compat_str
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_protocol,
|
determine_protocol,
|
||||||
|
NO_DEFAULT
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _get_real_downloader(info_dict, protocol=None, *args, **kwargs):
|
def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None):
|
||||||
|
info_dict['protocol'] = determine_protocol(info_dict)
|
||||||
info_copy = info_dict.copy()
|
info_copy = info_dict.copy()
|
||||||
if protocol:
|
if protocol:
|
||||||
info_copy['protocol'] = protocol
|
info_copy['protocol'] = protocol
|
||||||
return get_suitable_downloader(info_copy, *args, **kwargs)
|
return _get_suitable_downloader(info_copy, params, default)
|
||||||
|
|
||||||
|
|
||||||
# Some of these require _get_real_downloader
|
# Some of these require get_suitable_downloader
|
||||||
from .common import FileDownloader
|
from .common import FileDownloader
|
||||||
from .dash import DashSegmentsFD
|
from .dash import DashSegmentsFD
|
||||||
from .f4m import F4mFD
|
from .f4m import F4mFD
|
||||||
@ -69,14 +71,15 @@ def shorten_protocol_name(proto, simplify=False):
|
|||||||
return short_protocol_names.get(proto, proto)
|
return short_protocol_names.get(proto, proto)
|
||||||
|
|
||||||
|
|
||||||
def get_suitable_downloader(info_dict, params={}, default=HttpFD):
|
def _get_suitable_downloader(info_dict, params, default):
|
||||||
"""Get the downloader class that can handle the info dict."""
|
"""Get the downloader class that can handle the info dict."""
|
||||||
protocol = determine_protocol(info_dict)
|
if default is NO_DEFAULT:
|
||||||
info_dict['protocol'] = protocol
|
default = HttpFD
|
||||||
|
|
||||||
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
|
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
|
||||||
# return FFmpegFD
|
# return FFmpegFD
|
||||||
|
|
||||||
|
protocol = info_dict['protocol']
|
||||||
downloaders = params.get('external_downloader')
|
downloaders = params.get('external_downloader')
|
||||||
external_downloader = (
|
external_downloader = (
|
||||||
downloaders if isinstance(downloaders, compat_str) or downloaders is None
|
downloaders if isinstance(downloaders, compat_str) or downloaders is None
|
||||||
@ -94,7 +97,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
|
|||||||
return FFmpegFD
|
return FFmpegFD
|
||||||
elif external_downloader == 'native':
|
elif external_downloader == 'native':
|
||||||
return HlsFD
|
return HlsFD
|
||||||
elif _get_real_downloader(info_dict, 'm3u8_frag_urls', params, None):
|
elif get_suitable_downloader(info_dict, params, None, protocol='m3u8_frag_urls'):
|
||||||
return HlsFD
|
return HlsFD
|
||||||
elif params.get('hls_prefer_native') is True:
|
elif params.get('hls_prefer_native') is True:
|
||||||
return HlsFD
|
return HlsFD
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from ..downloader import _get_real_downloader
|
from ..downloader import get_suitable_downloader
|
||||||
from .fragment import FragmentFD
|
from .fragment import FragmentFD
|
||||||
|
|
||||||
from ..utils import urljoin
|
from ..utils import urljoin
|
||||||
@ -15,11 +15,14 @@ class DashSegmentsFD(FragmentFD):
|
|||||||
FD_NAME = 'dashsegments'
|
FD_NAME = 'dashsegments'
|
||||||
|
|
||||||
def real_download(self, filename, info_dict):
|
def real_download(self, filename, info_dict):
|
||||||
|
if info_dict.get('is_live'):
|
||||||
|
self.report_error('Live DASH videos are not supported')
|
||||||
|
|
||||||
fragment_base_url = info_dict.get('fragment_base_url')
|
fragment_base_url = info_dict.get('fragment_base_url')
|
||||||
fragments = info_dict['fragments'][:1] if self.params.get(
|
fragments = info_dict['fragments'][:1] if self.params.get(
|
||||||
'test', False) else info_dict['fragments']
|
'test', False) else info_dict['fragments']
|
||||||
|
|
||||||
real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None)
|
real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='dash_frag_urls')
|
||||||
|
|
||||||
ctx = {
|
ctx = {
|
||||||
'filename': filename,
|
'filename': filename,
|
||||||
@ -54,9 +57,6 @@ def real_download(self, filename, info_dict):
|
|||||||
info_copy = info_dict.copy()
|
info_copy = info_dict.copy()
|
||||||
info_copy['fragments'] = fragments_to_download
|
info_copy['fragments'] = fragments_to_download
|
||||||
fd = real_downloader(self.ydl, self.params)
|
fd = real_downloader(self.ydl, self.params)
|
||||||
# TODO: Make progress updates work without hooking twice
|
|
||||||
# for ph in self._progress_hooks:
|
|
||||||
# fd.add_progress_hook(ph)
|
|
||||||
return fd.real_download(filename, info_copy)
|
return fd.real_download(filename, info_copy)
|
||||||
|
|
||||||
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)
|
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)
|
||||||
|
@ -345,12 +345,22 @@ class FFmpegFD(ExternalFD):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def available(cls, path=None):
|
def available(cls, path=None):
|
||||||
# TODO: Fix path for ffmpeg
|
# TODO: Fix path for ffmpeg
|
||||||
|
# Fixme: This may be wrong when --ffmpeg-location is used
|
||||||
return FFmpegPostProcessor().available
|
return FFmpegPostProcessor().available
|
||||||
|
|
||||||
def on_process_started(self, proc, stdin):
|
def on_process_started(self, proc, stdin):
|
||||||
""" Override this in subclasses """
|
""" Override this in subclasses """
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def can_merge_formats(cls, info_dict, params={}):
|
||||||
|
return (
|
||||||
|
info_dict.get('requested_formats')
|
||||||
|
and info_dict.get('protocol')
|
||||||
|
and not params.get('allow_unplayable_formats')
|
||||||
|
and 'no-direct-merge' not in params.get('compat_opts', [])
|
||||||
|
and cls.can_download(info_dict))
|
||||||
|
|
||||||
def _call_downloader(self, tmpfilename, info_dict):
|
def _call_downloader(self, tmpfilename, info_dict):
|
||||||
urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
|
urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
|
||||||
ffpp = FFmpegPostProcessor(downloader=self)
|
ffpp = FFmpegPostProcessor(downloader=self)
|
||||||
|
@ -4,7 +4,7 @@
|
|||||||
import io
|
import io
|
||||||
import binascii
|
import binascii
|
||||||
|
|
||||||
from ..downloader import _get_real_downloader
|
from ..downloader import get_suitable_downloader
|
||||||
from .fragment import FragmentFD, can_decrypt_frag
|
from .fragment import FragmentFD, can_decrypt_frag
|
||||||
from .external import FFmpegFD
|
from .external import FFmpegFD
|
||||||
|
|
||||||
@ -80,16 +80,13 @@ def real_download(self, filename, info_dict):
|
|||||||
fd = FFmpegFD(self.ydl, self.params)
|
fd = FFmpegFD(self.ydl, self.params)
|
||||||
self.report_warning(
|
self.report_warning(
|
||||||
'%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename()))
|
'%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename()))
|
||||||
# TODO: Make progress updates work without hooking twice
|
|
||||||
# for ph in self._progress_hooks:
|
|
||||||
# fd.add_progress_hook(ph)
|
|
||||||
return fd.real_download(filename, info_dict)
|
return fd.real_download(filename, info_dict)
|
||||||
|
|
||||||
is_webvtt = info_dict['ext'] == 'vtt'
|
is_webvtt = info_dict['ext'] == 'vtt'
|
||||||
if is_webvtt:
|
if is_webvtt:
|
||||||
real_downloader = None # Packing the fragments is not currently supported for external downloader
|
real_downloader = None # Packing the fragments is not currently supported for external downloader
|
||||||
else:
|
else:
|
||||||
real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
|
real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='m3u8_frag_urls')
|
||||||
if real_downloader and not real_downloader.supports_manifest(s):
|
if real_downloader and not real_downloader.supports_manifest(s):
|
||||||
real_downloader = None
|
real_downloader = None
|
||||||
if real_downloader:
|
if real_downloader:
|
||||||
|
@ -4,7 +4,7 @@
|
|||||||
import threading
|
import threading
|
||||||
|
|
||||||
from .common import FileDownloader
|
from .common import FileDownloader
|
||||||
from ..downloader import _get_real_downloader
|
from ..downloader import get_suitable_downloader
|
||||||
from ..extractor.niconico import NiconicoIE
|
from ..extractor.niconico import NiconicoIE
|
||||||
from ..compat import compat_urllib_request
|
from ..compat import compat_urllib_request
|
||||||
|
|
||||||
@ -20,7 +20,7 @@ def real_download(self, filename, info_dict):
|
|||||||
ie = NiconicoIE(self.ydl)
|
ie = NiconicoIE(self.ydl)
|
||||||
info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
|
info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
|
||||||
|
|
||||||
fd = _get_real_downloader(info_dict, params=self.params)(self.ydl, self.params)
|
fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
|
||||||
|
|
||||||
success = download_complete = False
|
success = download_complete = False
|
||||||
timer = [None]
|
timer = [None]
|
||||||
|
@ -1298,7 +1298,7 @@ def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
|
|||||||
# JSON-LD may be malformed and thus `fatal` should be respected.
|
# JSON-LD may be malformed and thus `fatal` should be respected.
|
||||||
# At the same time `default` may be passed that assumes `fatal=False`
|
# At the same time `default` may be passed that assumes `fatal=False`
|
||||||
# for _search_regex. Let's simulate the same behavior here as well.
|
# for _search_regex. Let's simulate the same behavior here as well.
|
||||||
fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
|
fatal = kwargs.get('fatal', True) if default is NO_DEFAULT else False
|
||||||
json_ld = []
|
json_ld = []
|
||||||
for mobj in json_ld_list:
|
for mobj in json_ld_list:
|
||||||
json_ld_item = self._parse_json(
|
json_ld_item = self._parse_json(
|
||||||
|
@ -706,9 +706,8 @@ def _dict_from_options_callback(
|
|||||||
callback_kwargs={
|
callback_kwargs={
|
||||||
'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
|
'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
|
||||||
'default_key': 'default',
|
'default_key': 'default',
|
||||||
'process': lambda x: x.strip()
|
'process': str.strip
|
||||||
},
|
}, help=(
|
||||||
help=(
|
|
||||||
'Name or path of the external downloader to use (optionally) prefixed by '
|
'Name or path of the external downloader to use (optionally) prefixed by '
|
||||||
'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
|
'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
|
||||||
'Currently supports native, %s (Recommended: aria2c). '
|
'Currently supports native, %s (Recommended: aria2c). '
|
||||||
@ -724,8 +723,7 @@ def _dict_from_options_callback(
|
|||||||
'allowed_keys': '|'.join(list_external_downloaders()),
|
'allowed_keys': '|'.join(list_external_downloaders()),
|
||||||
'default_key': 'default',
|
'default_key': 'default',
|
||||||
'process': compat_shlex_split
|
'process': compat_shlex_split
|
||||||
},
|
}, help=(
|
||||||
help=(
|
|
||||||
'Give these arguments to the external downloader. '
|
'Give these arguments to the external downloader. '
|
||||||
'Specify the downloader name and the arguments separated by a colon ":". '
|
'Specify the downloader name and the arguments separated by a colon ":". '
|
||||||
'You can use this option multiple times to give different arguments to different downloaders '
|
'You can use this option multiple times to give different arguments to different downloaders '
|
||||||
@ -944,8 +942,7 @@ def _dict_from_options_callback(
|
|||||||
callback_kwargs={
|
callback_kwargs={
|
||||||
'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()),
|
'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()),
|
||||||
'default_key': 'default'
|
'default_key': 'default'
|
||||||
},
|
}, help='Output filename template; see "OUTPUT TEMPLATE" for details')
|
||||||
help='Output filename template; see "OUTPUT TEMPLATE" for details')
|
|
||||||
filesystem.add_option(
|
filesystem.add_option(
|
||||||
'--output-na-placeholder',
|
'--output-na-placeholder',
|
||||||
dest='outtmpl_na_placeholder', metavar='TEXT', default='NA',
|
dest='outtmpl_na_placeholder', metavar='TEXT', default='NA',
|
||||||
@ -1191,8 +1188,7 @@ def _dict_from_options_callback(
|
|||||||
'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat',
|
'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat',
|
||||||
'process': compat_shlex_split,
|
'process': compat_shlex_split,
|
||||||
'multiple_keys': False
|
'multiple_keys': False
|
||||||
},
|
}, help=(
|
||||||
help=(
|
|
||||||
'Give these arguments to the postprocessors. '
|
'Give these arguments to the postprocessors. '
|
||||||
'Specify the postprocessor/executable name and the arguments separated by a colon ":" '
|
'Specify the postprocessor/executable name and the arguments separated by a colon ":" '
|
||||||
'to give the argument to the specified postprocessor/executable. Supported PP are: '
|
'to give the argument to the specified postprocessor/executable. Supported PP are: '
|
||||||
@ -1385,8 +1381,7 @@ def _dict_from_options_callback(
|
|||||||
'multiple_keys': False,
|
'multiple_keys': False,
|
||||||
'process': lambda val: dict(
|
'process': lambda val: dict(
|
||||||
_extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';'))
|
_extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';'))
|
||||||
},
|
}, help=(
|
||||||
help=(
|
|
||||||
'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. '
|
'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. '
|
||||||
'You can use this option multiple times to give arguments for different extractors'))
|
'You can use this option multiple times to give arguments for different extractors'))
|
||||||
extractor.add_option(
|
extractor.add_option(
|
||||||
|
@ -4544,7 +4544,7 @@ def parse_codecs(codecs_str):
|
|||||||
if not codecs_str:
|
if not codecs_str:
|
||||||
return {}
|
return {}
|
||||||
split_codecs = list(filter(None, map(
|
split_codecs = list(filter(None, map(
|
||||||
lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
|
str.strip, codecs_str.strip().strip(',').split(','))))
|
||||||
vcodec, acodec = None, None
|
vcodec, acodec = None, None
|
||||||
for full_codec in split_codecs:
|
for full_codec in split_codecs:
|
||||||
codec = full_codec.split('.')[0]
|
codec = full_codec.split('.')[0]
|
||||||
@ -6246,7 +6246,7 @@ def traverse_obj(
|
|||||||
# TODO: Write tests
|
# TODO: Write tests
|
||||||
'''
|
'''
|
||||||
if not casesense:
|
if not casesense:
|
||||||
_lower = lambda k: k.lower() if isinstance(k, str) else k
|
_lower = lambda k: (k.lower() if isinstance(k, str) else k)
|
||||||
path_list = (map(_lower, variadic(path)) for path in path_list)
|
path_list = (map(_lower, variadic(path)) for path in path_list)
|
||||||
|
|
||||||
def _traverse_obj(obj, path, _current_depth=0):
|
def _traverse_obj(obj, path, _current_depth=0):
|
||||||
|
Loading…
Reference in New Issue
Block a user