From 251ae04e6a057167e4eafaf8b7b565a984b48405 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 23 Aug 2021 04:45:30 +0530 Subject: [PATCH] [lazy_extractor] Create instance only after pre-checking archive --- devscripts/lazy_load_template.py | 1 + devscripts/make_lazy_extractors.py | 15 ++++++++------- yt_dlp/YoutubeDL.py | 6 +++--- yt_dlp/__init__.py | 4 ++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index a3f3fedf9..036e2e767 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -9,6 +9,7 @@ def __getattr__(cls, name): class LazyLoadExtractor(metaclass=LazyLoadMetaClass): _module = None + _WORKING = True @classmethod def _get_real_class(cls): diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 8189c77dc..d313e68a9 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -30,16 +30,14 @@ with open('devscripts/lazy_load_template.py', 'rt') as f: module_template = f.read() +CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] module_contents = [ module_template, - getsource(InfoExtractor.ie_key), - getsource(InfoExtractor._match_valid_url), - getsource(InfoExtractor.suitable), + *[getsource(getattr(InfoExtractor, k)) for k in CLASS_PROPERTIES], '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] ie_template = ''' class {name}({bases}): - _VALID_URL = {valid_url!r} _module = '{module}' ''' @@ -60,14 +58,17 @@ def get_base_name(base): def build_lazy_ie(ie, name): - valid_url = getattr(ie, '_VALID_URL', None) s = ie_template.format( name=name, bases=', '.join(map(get_base_name, ie.__bases__)), - valid_url=valid_url, module=ie.__module__) + valid_url = getattr(ie, '_VALID_URL', None) + if valid_url: + s += f' _VALID_URL = {valid_url!r}\n' + if not ie._WORKING: + s += f' _WORKING = False\n' if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += '\n' + getsource(ie.suitable) + s += f'\n{getsource(ie.suitable)}' if hasattr(ie, '_make_valid_url'): # search extractors s += make_valid_template.format(valid_url=ie._make_valid_url()) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 17aa63487..5d40eb333 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -1179,7 +1179,7 @@ def extract_info(self, url, download=True, ie_key=None, extra_info={}, ie_key = 'Generic' if ie_key: - ies = [self.get_info_extractor(ie_key)] + ies = [get_info_extractor(ie_key)] else: ies = self._ies @@ -1188,7 +1188,6 @@ def extract_info(self, url, download=True, ie_key=None, extra_info={}, continue ie_key = ie.ie_key() - ie = self.get_info_extractor(ie_key) if not ie.working(): self.report_warning('The program functionality for this site has been marked as broken, ' 'and will probably not work.') @@ -1198,7 +1197,8 @@ def extract_info(self, url, download=True, ie_key=None, extra_info={}, self.to_screen("[%s] %s: has already been recorded in archive" % ( ie_key, temp_id)) break - return self.__extract_info(url, ie, download, extra_info, process) + return self.__extract_info(url, self.get_info_extractor(ie.ie_key()), + download, extra_info, process) else: self.report_error('no suitable InfoExtractor for URL %s' % url) diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 36b2eddba..58e8ea5d9 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -110,14 +110,14 @@ def _real_main(argv=None): if opts.list_extractors: for ie in list_extractors(opts.age_limit): - write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '') + '\n', out=sys.stdout) + write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie.working() else '') + '\n', out=sys.stdout) matchedUrls = [url for url in all_urls if ie.suitable(url)] for mu in matchedUrls: write_string(' ' + mu + '\n', out=sys.stdout) sys.exit(0) if opts.list_extractor_descriptions: for ie in list_extractors(opts.age_limit): - if not ie._WORKING: + if not ie.working(): continue desc = getattr(ie, 'IE_DESC', ie.IE_NAME) if desc is False: