Mirrors/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-05 18:52:48 +01:00
Code Issues Releases Activity

[cleanup] Misc fixes

Browse Source 
Closes #3565, https://github.com/yt-dlp/yt-dlp/issues/3514#issuecomment-1105944364
This commit is contained in:
pukkandan 2022-04-29 07:18:36 +05:30
parent 0a41f331cc
commit 1d485a1a79
No known key found for this signature in database
GPG Key ID: 7EEE9E1E817D0A39
19 changed files with 75 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
devscripts/lazy_load_template.py
View File

@ -7,7 +7,7 @@ class LazyLoadMetaClass(type):
def __getattr__(cls, name):
if '_real_class' not in cls.__dict__:
write_string(
f'WARNING: Falling back to normal extractor since lazy extractor '
'WARNING: Falling back to normal extractor since lazy extractor '
f'{cls.__name__} does not have attribute {name}{bug_reports_message()}')
return getattr(cls._get_real_class(), name)

18
yt_dlp/YoutubeDL.py
View File

@ -62,6 +62,7 @@
DEFAULT_OUTTMPL,
LINK_TEMPLATES,
NO_DEFAULT,
NUMBER_RE,
OUTTMPL_TYPES,
POSTPROCESS_WHEN,
STR_FORMAT_RE_TMPL,
@ -1049,7 +1050,7 @@ def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
formatSeconds(info_dict['duration'], '-' if sanitize else ':')
if info_dict.get('duration', None) is not None
else None)
info_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
info_dict['autonumber'] = int(self.params.get('autonumber_start', 1) - 1 + self._num_downloads)
info_dict['video_autonumber'] = self._num_videos
if info_dict.get('resolution') is None:
info_dict['resolution'] = self.format_resolution(info_dict, default=None)
@ -1071,18 +1072,18 @@ def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
# Field is of the form key1.key2...
# where keys (except first) can be string, int or slice
FIELD_RE = r'\w*(?:\.(?:\w+|{num}|{num}?(?::{num}?){{1,2}}))*'.format(num=r'(?:-?\d+)')
MATH_FIELD_RE = r'''(?:{field}|{num})'''.format(field=FIELD_RE, num=r'-?\d+(?:.\d+)?')
MATH_FIELD_RE = rf'(?:{FIELD_RE}|-?{NUMBER_RE})'
MATH_OPERATORS_RE = r'(?:%s)' % '|'.join(map(re.escape, MATH_FUNCTIONS.keys()))
INTERNAL_FORMAT_RE = re.compile(r'''(?x)
INTERNAL_FORMAT_RE = re.compile(rf'''(?x)
(?P<negate>-)?
(?P<fields>{field})
(?P<maths>(?:{math_op}{math_field})*)
(?P<fields>{FIELD_RE})
(?P<maths>(?:{MATH_OPERATORS_RE}{MATH_FIELD_RE})*)
(?:>(?P<strf_format>.+?))?
(?P<remaining>
(?P<alternate>(?<!\\),[^|&)]+)?
(?:&(?P<replacement>.*?))?
(?:\|(?P<default>.*?))?
)$'''.format(field=FIELD_RE, math_op=MATH_OPERATORS_RE, math_field=MATH_FIELD_RE))
)$''')
def _traverse_infodict(k):
k = k.split('.')
@ -2336,7 +2337,7 @@ def _fill_common_fields(self, info_dict, is_video=True):
video_id=info_dict['id'], ie=info_dict['extractor'])
elif not info_dict.get('title'):
self.report_warning('Extractor failed to obtain "title". Creating a generic title instead')
info_dict['title'] = f'{info_dict["extractor"]} video #{info_dict["id"]}'
info_dict['title'] = f'{info_dict["extractor"].replace(":", "-")} video #{info_dict["id"]}'
if info_dict.get('duration') is not None:
info_dict['duration_string'] = formatSeconds(info_dict['duration'])
@ -3669,10 +3670,11 @@ def python_implementation():
) or 'none'
write_debug('exe versions: %s' % exe_str)
from .compat.compat_utils import get_package_info
from .dependencies import available_dependencies
write_debug('Optional libraries: %s' % (', '.join(sorted({
module.__name__.split('.')[0] for module in available_dependencies.values()
join_nonempty(*get_package_info(m)) for m in available_dependencies.values()
})) or 'none'))
self._setup_opener()

4
yt_dlp/compat/__init__.py
View File

@ -46,10 +46,6 @@ def compat_ord(c):
return c if isinstance(c, int) else ord(c)
def compat_setenv(key, value, env=os.environ):
env[key] = value
if compat_os_name == 'nt' and sys.version_info < (3, 8):
# os.path.realpath on Windows does not follow symbolic links
# prior to Python 3.8 (see https://bugs.python.org/issue9949)

5
yt_dlp/compat/_deprecated.py
View File

@ -44,4 +44,9 @@
compat_urllib_request = urllib.request
compat_urlparse = compat_urllib_parse = urllib.parse
def compat_setenv(key, value, env=os.environ):
env[key] = value
__all__ = [x for x in globals() if x.startswith('compat_')]

1
yt_dlp/compat/asyncio.py
View File

@ -1,5 +1,4 @@
# flake8: noqa: F405
from asyncio import * # noqa: F403
from .compat_utils import passthrough_module

22
yt_dlp/compat/compat_utils.py
View File

@ -1,9 +1,28 @@
import collections
import contextlib
import importlib
import sys
import types
_NO_ATTRIBUTE = object()
_Package = collections.namedtuple('Package', ('name', 'version'))
def get_package_info(module):
parent = module.__name__.split('.')[0]
parent_module = None
with contextlib.suppress(ImportError):
parent_module = importlib.import_module(parent)
for attr in ('__version__', 'version_string', 'version'):
version = getattr(parent_module, attr, None)
if version is not None:
break
return _Package(getattr(module, '_yt_dlp__identifier', parent), str(version))
def _is_package(module):
try:
module.__getattribute__('__path__')
@ -12,9 +31,6 @@ def _is_package(module):
return True
_NO_ATTRIBUTE = object()
def passthrough_module(parent, child, *, callback=lambda _: None):
parent_module = importlib.import_module(parent)
child_module = importlib.import_module(child, parent)

1
yt_dlp/compat/re.py
View File

@ -1,5 +1,4 @@
# flake8: noqa: F405
from re import * # F403
from .compat_utils import passthrough_module

11
yt_dlp/dependencies.py
View File

@ -1,4 +1,6 @@
# flake8: noqa: F401
"""Imports all optional dependencies for the project.
An attribute "_yt_dlp__identifier" may be inserted into the module if it uses an ambigious namespace"""
try:
import brotlicffi as brotli
@ -28,6 +30,15 @@
from Crypto.Cipher import AES as Cryptodome_AES
except ImportError:
Cryptodome_AES = None
else:
try:
# In pycrypto, mode defaults to ECB. See:
# https://www.pycryptodome.org/en/latest/src/vs_pycrypto.html#:~:text=not%20have%20ECB%20as%20default%20mode
Cryptodome_AES.new(b'abcdefghijklmnop')
except TypeError:
pass
else:
Cryptodome_AES._yt_dlp__identifier = 'pycrypto'
try:

6
yt_dlp/downloader/common.py
View File

@ -12,6 +12,7 @@
QuietMultilinePrinter,
)
from ..utils import (
NUMBER_RE,
LockingUnsupportedError,
Namespace,
decodeArgument,
@ -91,7 +92,8 @@ def _set_ydl(self, ydl):
'trouble',
'write_debug',
):
setattr(self, func, getattr(ydl, func))
if not hasattr(self, func):
setattr(self, func, getattr(ydl, func))
def to_screen(self, *args, **kargs):
self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs)
@ -170,7 +172,7 @@ def best_block_size(elapsed_time, bytes):
@staticmethod
def parse_bytes(bytestr):
"""Parse a string indicating a byte quantity into an integer."""
matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
matchobj = re.match(rf'(?i)^({NUMBER_RE})([kMGTPEZY]?)$', bytestr)
if matchobj is None:
return None
number = float(matchobj.group(1))

2
yt_dlp/downloader/external.py
View File

@ -368,7 +368,7 @@ def _call_downloader(self, tmpfilename, info_dict):
# These exists only for compatibility. Extractors should use
# info_dict['downloader_options']['ffmpeg_args'] instead
args += info_dict.get('_ffmpeg_args')
args += info_dict.get('_ffmpeg_args') or []
seekable = info_dict.get('_seekable')
if seekable is not None:
# setting -seekable prevents ffmpeg from guessing if the server

12
yt_dlp/downloader/fragment.py
View File

@ -1,3 +1,4 @@
import concurrent.futures
import contextlib
import http.client
import json
@ -5,12 +6,6 @@
import os
import time
try:
import concurrent.futures
can_threaded_download = True
except ImportError:
can_threaded_download = False
from .common import FileDownloader
from .http import HttpFD
from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
@ -28,6 +23,8 @@ class HttpQuietDownloader(HttpFD):
def to_screen(self, *args, **kargs):
pass
console_title = to_screen
def report_retry(self, err, count, retries):
super().to_screen(
f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...')
@ -501,8 +498,7 @@ def append_fragment(frag_content, frag_index, ctx):
max_workers = math.ceil(
self.params.get('concurrent_fragment_downloads', 1) / ctx.get('max_progress', 1))
if can_threaded_download and max_workers > 1:
if max_workers > 1:
def _download_fragment(fragment):
ctx_copy = ctx.copy()
download_fragment(fragment, ctx_copy)

2
yt_dlp/downloader/mhtml.py
View File

@ -173,7 +173,7 @@ def real_download(self, filename, info_dict):
mime_type = b'image/png'
if frag_content.startswith((b'GIF87a', b'GIF89a')):
mime_type = b'image/gif'
if frag_content.startswith(b'RIFF') and frag_content[8:12] == 'WEBP':
if frag_content.startswith(b'RIFF') and frag_content[8:12] == b'WEBP':
mime_type = b'image/webp'
frag_header = io.BytesIO()

3
yt_dlp/extractor/common.py
View File

@ -1922,8 +1922,7 @@ def calculate_preference(self, format):
def _sort_formats(self, formats, field_preference=[]):
if not formats:
return
format_sort = self.FormatSort(self, field_preference)
formats.sort(key=lambda f: format_sort.calculate_preference(f))
formats.sort(key=self.FormatSort(self, field_preference).calculate_preference)
def _check_formats(self, formats, video_id):
if formats:

6
yt_dlp/extractor/fujitv.py
View File

@ -17,7 +17,7 @@ class FujiTVFODPlus7IE(InfoExtractor):
'url': 'https://fod.fujitv.co.jp/title/5d40/5d40110076',
'info_dict': {
'id': '5d40110076',
'ext': 'mp4',
'ext': 'ts',
'title': '#1318 『まる子、まぼろしの洋館を見る』の巻',
'series': 'ちびまる子ちゃん',
'series_id': '5d40',
@ -28,7 +28,7 @@ class FujiTVFODPlus7IE(InfoExtractor):
'url': 'https://fod.fujitv.co.jp/title/5d40/5d40810083',
'info_dict': {
'id': '5d40810083',
'ext': 'mp4',
'ext': 'ts',
'title': '#1324 『まる子とオニの子』の巻／『結成！2月をムダにしない会』の巻',
'description': 'md5:3972d900b896adc8ab1849e310507efa',
'series': 'ちびまる子ちゃん',
@ -51,7 +51,7 @@ def _real_extract(self, url):
for src in src_json['video_selector']:
if not src.get('url'):
continue
fmt, subs = self._extract_m3u8_formats_and_subtitles(src['url'], video_id, 'mp4')
fmt, subs = self._extract_m3u8_formats_and_subtitles(src['url'], video_id, 'ts')
for f in fmt:
f.update(dict(zip(('height', 'width'),
self._BITRATE_MAP.get(f.get('tbr'), ()))))

3
yt_dlp/extractor/funimation.py
View File

@ -242,6 +242,9 @@ def _real_extract(self, url):
'language_preference': language_preference(lang.lower()),
})
formats.extend(current_formats)
if not formats and (requested_languages or requested_versions):
self.raise_no_formats(
'There are no video formats matching the requested languages/versions', expected=True, video_id=display_id)
self._remove_duplicate_formats(formats)
self._sort_formats(formats, ('lang', 'source'))

2
yt_dlp/extractor/youtube.py
View File

@ -3107,7 +3107,7 @@ def _extract_formats(self, streaming_data, video_id, player_url, is_live, durati
'n': self._decrypt_nsig(query['n'][0], video_id, player_url)})
except ExtractorError as e:
self.report_warning(
f'nsig extraction failed: You may experience throttling for some formats\n'
'nsig extraction failed: You may experience throttling for some formats\n'
f'n = {query["n"][0]} ; player = {player_url}\n{e}', only_once=True)
throttled = True

4
yt_dlp/postprocessor/embedthumbnail.py
View File

@ -79,9 +79,9 @@ def run(self, info):
original_thumbnail = thumbnail_filename = info['thumbnails'][idx]['filepath']
thumbnail_ext = os.path.splitext(thumbnail_filename)[1][1:]
# Convert unsupported thumbnail formats (see #25687, #25717)
# PNG is preferred since JPEG is lossy
thumbnail_ext = os.path.splitext(thumbnail_filename)[1][1:]
if info['ext'] not in ('mkv', 'mka') and thumbnail_ext not in ('jpg', 'jpeg', 'png'):
thumbnail_filename = convertor.convert_thumbnail(thumbnail_filename, 'png')
thumbnail_ext = 'png'
@ -100,7 +100,7 @@ def run(self, info):
elif info['ext'] in ['mkv', 'mka']:
options = list(self.stream_copy_opts())
mimetype = 'image/%s' % ('jpeg' if thumbnail_ext in ('jpg', 'jpeg') else thumbnail_ext)
mimetype = f'image/{thumbnail_ext.replace("jpg", "jpeg")}'
old_stream, new_stream = self.get_stream_number(
filename, ('tags', 'mimetype'), mimetype)
if old_stream is not None:

9
yt_dlp/postprocessor/xattrpp.py
View File

@ -1,3 +1,5 @@
import os
from .common import PostProcessor
from ..compat import compat_os_name
from ..utils import (
@ -28,6 +30,7 @@ def run(self, info):
self.to_screen('Writing metadata to file\'s xattrs')
filename = info['filepath']
mtime = os.stat(filename).st_mtime
try:
xattr_mapping = {
@ -53,8 +56,6 @@ def run(self, info):
write_xattr(filename, xattrname, byte_value)
num_written += 1
return [], info
except XAttrUnavailableError as e:
raise PostProcessingError(str(e))
@ -73,4 +74,6 @@ def run(self, info):
else:
msg += '(You may have to enable them in your /etc/fstab)'
raise PostProcessingError(str(e))
return [], info
self.try_utime(filename, mtime, mtime)
return [], info

4
yt_dlp/utils.py
View File

@ -245,6 +245,8 @@ def random_user_agent():
PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
NUMBER_RE = r'\d+(?:\.\d+)?'
def preferredencoding():
"""Get preferred encoding.
@ -3427,7 +3429,7 @@ def parse_dfxp_time_expr(time_expr):
if not time_expr:
return
mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
mobj = re.match(rf'^(?P<time_offset>{NUMBER_RE})s?$', time_expr)
if mobj:
return float(mobj.group('time_offset'))