From b868936cd6d619f0e3a2c303028c132b761e6313 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 7 Aug 2021 16:50:46 +0530 Subject: [PATCH] [cleanup] Misc --- devscripts/run_tests.sh | 1 - test/test_overwrites.py | 3 ++- yt_dlp/YoutubeDL.py | 45 ++++++++++++++++++------------------ yt_dlp/downloader/common.py | 7 ++---- yt_dlp/extractor/common.py | 11 +++++---- yt_dlp/extractor/niconico.py | 6 ++--- yt_dlp/utils.py | 7 ++++-- 7 files changed, 41 insertions(+), 39 deletions(-) diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index 99ab0a793..fb405b569 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -11,5 +11,4 @@ else exit 1 fi -echo python3 -m pytest -k $test_set python3 -m pytest -k "$test_set" diff --git a/test/test_overwrites.py b/test/test_overwrites.py index 20dfcb70a..f5d10a409 100644 --- a/test/test_overwrites.py +++ b/test/test_overwrites.py @@ -8,13 +8,14 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import try_rm +from test.helper import is_download_test, try_rm root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) download_file = join(root_dir, 'test.webm') +@is_download_test class TestOverwrites(unittest.TestCase): def setUp(self): # create an empty file diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index ac99dd45b..f4333c7e5 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -515,8 +515,15 @@ def check_deprecated(param, option, suggestion): self.report_warning('--merge-output-format will be ignored since --remux-video or --recode-video is given') self.params['merge_output_format'] = self.params['final_ext'] - if 'overwrites' in self.params and self.params['overwrites'] is None: - del self.params['overwrites'] + if self.params.get('overwrites') is None: + self.params.pop('overwrites', None) + elif self.params.get('nooverwrites') is not None: + # nooverwrites was unnecessarily changed to overwrites + # in 0c3d0f51778b153f65c21906031c2e091fcfb641 + # This ensures compatibility with both keys + self.params['overwrites'] = not self.params['nooverwrites'] + else: + self.params['nooverwrites'] = not self.params['overwrites'] if params.get('bidi_workaround', False): try: @@ -889,7 +896,6 @@ def validate_outtmpl(cls, outtmpl): def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None): """ Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """ info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set - na = self.params.get('outtmpl_na_placeholder', 'NA') info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList for key in ('__original_infodict', '__postprocessors'): @@ -970,6 +976,8 @@ def get_value(mdict): return value + na = self.params.get('outtmpl_na_placeholder', 'NA') + def _dumpjson_default(obj): if isinstance(obj, (set, LazyList)): return list(obj) @@ -978,10 +986,7 @@ def _dumpjson_default(obj): def create_key(outer_mobj): if not outer_mobj.group('has_key'): return f'%{outer_mobj.group(0)}' - - prefix = outer_mobj.group('prefix') key = outer_mobj.group('key') - original_fmt = fmt = outer_mobj.group('format') mobj = re.match(INTERNAL_FORMAT_RE, key) if mobj is None: value, default, mobj = None, na, {'fields': ''} @@ -990,6 +995,7 @@ def create_key(outer_mobj): default = mobj['default'] if mobj['default'] is not None else na value = get_value(mobj) + fmt = outer_mobj.group('format') if fmt == 's' and value is not None and key in field_size_compat_map.keys(): fmt = '0{:d}d'.format(field_size_compat_map[key]) @@ -1021,9 +1027,9 @@ def create_key(outer_mobj): if fmt[-1] in 'csr': value = sanitize(mobj['fields'].split('.')[-1], value) - key = '%s\0%s' % (key.replace('%', '%\0'), original_fmt) + key = '%s\0%s' % (key.replace('%', '%\0'), outer_mobj.group('format')) TMPL_DICT[key] = value - return f'{prefix}%({key}){fmt}' + return '{prefix}%({key}){fmt}'.format(key=key, fmt=fmt, prefix=outer_mobj.group('prefix')) return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT @@ -1069,7 +1075,6 @@ def prepare_filename(self, info_dict, dir_type='', warn=False): self.report_warning('--paths is ignored when an outputting to stdout', only_once=True) elif os.path.isabs(filename): self.report_warning('--paths is ignored since an absolute path is given in output template', only_once=True) - self.__prepare_filename_warned = True if filename == '-' or not filename: return filename @@ -1348,15 +1353,12 @@ def process_ie_result(self, ie_result, download=True, extra_info={}): 'It needs to be updated.' % ie_result.get('extractor')) def _fixup(r): - self.add_extra_info( - r, - { - 'extractor': ie_result['extractor'], - 'webpage_url': ie_result['webpage_url'], - 'webpage_url_basename': url_basename(ie_result['webpage_url']), - 'extractor_key': ie_result['extractor_key'], - } - ) + self.add_extra_info(r, { + 'extractor': ie_result['extractor'], + 'webpage_url': ie_result['webpage_url'], + 'webpage_url_basename': url_basename(ie_result['webpage_url']), + 'extractor_key': ie_result['extractor_key'], + }) return r ie_result['entries'] = [ self.process_ie_result(_fixup(r), download, extra_info) @@ -2193,7 +2195,7 @@ def is_wellformed(f): format['format'] = '{id} - {res}{note}'.format( id=format['format_id'], res=self.format_resolution(format), - note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '', + note=format_field(format, 'format_note', ' (%s)'), ) # Automatically determine file extension if missing if format.get('ext') is None: @@ -2395,7 +2397,7 @@ def print_optional(field): print_optional('thumbnail') print_optional('description') print_optional('filename') - if self.params.get('forceduration', False) and info_dict.get('duration') is not None: + if self.params.get('forceduration') and info_dict.get('duration') is not None: self.to_stdout(formatSeconds(info_dict['duration'])) print_mandatory('format') @@ -2435,8 +2437,6 @@ def process_info(self, info_dict): assert info_dict.get('_type', 'video') == 'video' - info_dict.setdefault('__postprocessors', []) - max_downloads = self.params.get('max_downloads') if max_downloads is not None: if self._num_downloads >= int(max_downloads): @@ -2637,6 +2637,7 @@ def _write_link_file(extension, template, newline, embed_filename): info_dict = self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict) else: # Download + info_dict.setdefault('__postprocessors', []) try: def existing_file(*filepaths): diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 500d7a49e..396521aa1 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -320,12 +320,9 @@ def report_retry(self, err, count, retries): '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...' % (error_to_compat_str(err), count, self.format_retries(retries))) - def report_file_already_downloaded(self, file_name): + def report_file_already_downloaded(self, *args, **kwargs): """Report file has already been fully downloaded.""" - try: - self.to_screen('[download] %s has already been downloaded' % file_name) - except UnicodeEncodeError: - self.to_screen('[download] The file has already been downloaded') + return self.ydl.report_file_already_downloaded(*args, **kwargs) def report_unable_to_resume(self): """Report it was impossible to resume download.""" diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 25b6dbc50..d6fb56db2 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -35,7 +35,6 @@ remove_encrypted_media, ) from ..utils import ( - NO_DEFAULT, age_restricted, base_url, bug_reports_message, @@ -45,10 +44,11 @@ determine_protocol, dict_get, error_to_compat_str, - ExtractorError, extract_attributes, + ExtractorError, fix_xml_ampersands, float_or_none, + format_field, GeoRestrictedError, GeoUtils, int_or_none, @@ -56,6 +56,7 @@ JSON_LD_RE, mimetype2ext, network_exceptions, + NO_DEFAULT, orderedSet, parse_bitrate, parse_codecs, @@ -64,8 +65,8 @@ parse_m3u8_attributes, parse_resolution, RegexNotFoundError, - sanitized_Request, sanitize_filename, + sanitized_Request, str_or_none, str_to_int, strip_or_none, @@ -75,9 +76,9 @@ unified_timestamp, update_Request, update_url_query, - urljoin, url_basename, url_or_none, + urljoin, variadic, xpath_element, xpath_text, @@ -1002,7 +1003,7 @@ def _download_socket_json( return res if res is False else res[0] def report_warning(self, msg, video_id=None, *args, only_once=False, **kwargs): - idstr = '' if video_id is None else '%s: ' % video_id + idstr = format_field(video_id, template='%s: ') msg = f'[{self.IE_NAME}] {idstr}{msg}' if only_once: if f'WARNING: {msg}' in self._printed_messages: diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py index 1c15dca71..2fa81b5c2 100644 --- a/yt_dlp/extractor/niconico.py +++ b/yt_dlp/extractor/niconico.py @@ -13,16 +13,16 @@ compat_urllib_parse_urlparse, ) from ..utils import ( - dict_get, ExtractorError, - int_or_none, + dict_get, float_or_none, + int_or_none, OnDemandPagedList, parse_duration, parse_iso8601, PostProcessingError, - str_or_none, remove_start, + str_or_none, try_get, unified_timestamp, urlencode_postdata, diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index fd13febd6..4d83b1fbe 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -6161,8 +6161,11 @@ def to_high_limit_path(path): return path -def format_field(obj, field, template='%s', ignore=(None, ''), default='', func=None): - val = obj.get(field, default) +def format_field(obj, field=None, template='%s', ignore=(None, ''), default='', func=None): + if field is None: + val = obj if obj is not None else default + else: + val = obj.get(field, default) if func and val not in ignore: val = func(val) return template % val if val not in ignore else default