Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-26 04:32:51 +01:00
Code Issues Releases Wiki Activity

optimize directory path generation

Browse Source 
- use str.join() instead of os.path.join()
  (less "features", but 10x as fast)
- cache directory formatters
- detect and optimize field access for 1-element format strings
This commit is contained in:
Mike Fährmann 2019-08-19 15:56:20 +02:00
parent 51d10783fc
commit e77a656437
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
1 changed files with 35 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
gallery_dl/util.py
View File

@ -391,10 +391,18 @@ class Formatter():
if field_name: if field_name:
self.fields.append(( self.fields.append((
len(self.result), len(self.result),
self._field_access(field_name, format_spec, conversion) self._field_access(field_name, format_spec, conversion),
)) ))
self.result.append("") self.result.append("")
if len(self.result) == 1:
if self.fields:
self.format_map = self.fields[0][1]
else:
self.format_map = lambda _: format_string
del self.result
del self.fields
def format_map(self, kwargs): def format_map(self, kwargs):
"""Apply 'kwargs' to the initial format_string and return its result""" """Apply 'kwargs' to the initial format_string and return its result"""
for index, func in self.fields: for index, func in self.fields:
@ -512,17 +520,24 @@ class Formatter():
class PathFormat(): class PathFormat():
def __init__(self, extractor): def __init__(self, extractor):
self.filename_fmt = extractor.config( filename_fmt = extractor.config("filename", extractor.filename_fmt)
"filename", extractor.filename_fmt) directory_fmt = extractor.config("directory", extractor.directory_fmt)
self.directory_fmt = extractor.config( kwdefault = extractor.config("keywords-default")
"directory", extractor.directory_fmt)
self.kwdefault = extractor.config("keywords-default")
try: try:
self.formatter = Formatter(self.filename_fmt, self.kwdefault) self.filename_formatter = Formatter(
filename_fmt, kwdefault).format_map
except Exception as exc: except Exception as exc:
raise exception.FormatError(exc, "filename") raise exception.FormatError(exc, "filename")
try:
self.directory_formatters = [
Formatter(dirfmt, kwdefault).format_map
for dirfmt in directory_fmt
]
except Exception as exc:
raise exception.FormatError(exc, "directory")
self.directory = self.realdirectory = "" self.directory = self.realdirectory = ""
self.filename = "" self.filename = ""
self.extension = "" self.extension = ""
@ -535,6 +550,8 @@ class PathFormat():
extractor.config("base-directory", (".", "gallery-dl"))) extractor.config("base-directory", (".", "gallery-dl")))
if os.altsep and os.altsep in self.basedirectory: if os.altsep and os.altsep in self.basedirectory:
self.basedirectory = self.basedirectory.replace(os.altsep, os.sep) self.basedirectory = self.basedirectory.replace(os.altsep, os.sep)
if self.basedirectory[-1] != os.sep:
self.basedirectory += os.sep
restrict = extractor.config("path-restrict", "auto") restrict = extractor.config("path-restrict", "auto")
if restrict == "auto": if restrict == "auto":
@ -592,30 +609,27 @@ class PathFormat():
# Build path segments by applying 'kwdict' to directory format strings # Build path segments by applying 'kwdict' to directory format strings
try: try:
segments = [ segments = [
self.clean_segment( self.clean_segment(format_map(kwdict).strip())
Formatter(segment, self.kwdefault) for format_map in self.directory_formatters
.format_map(kwdict)
.strip()
)
for segment in self.directory_fmt
] ]
except Exception as exc: except Exception as exc:
raise exception.FormatError(exc, "directory") raise exception.FormatError(exc, "directory")
# Join path segements # Join path segements
self.directory = self.clean_path(os.path.join( directory = self.clean_path(
self.basedirectory, *segments)) self.basedirectory + os.sep.join(segments))
# Remove trailing path separator; # Remove trailing path separator;
# occurs if the last argument to os.path.join() is an empty string # occurs if the last segment is an empty string
if self.directory[-1] == os.sep: if directory[-1] == os.sep:
self.directory = self.directory[:-1] directory = directory[:-1]
self.directory = directory
# Enable longer-than-260-character paths on Windows # Enable longer-than-260-character paths on Windows
if os.name == "nt": if os.name == "nt":
self.realdirectory = "\\\\?\\" + os.path.abspath(self.directory) self.realdirectory = "\\\\?\\" + os.path.abspath(directory)
else: else:
self.realdirectory = self.directory self.realdirectory = directory
# Create directory tree # Create directory tree
os.makedirs(self.realdirectory, exist_ok=True) os.makedirs(self.realdirectory, exist_ok=True)
@ -651,7 +665,7 @@ class PathFormat():
# Apply 'kwdict' to filename format string # Apply 'kwdict' to filename format string
try: try:
self.filename = self.clean_path(self.clean_segment( self.filename = self.clean_path(self.clean_segment(
self.formatter.format_map(self.kwdict))) self.filename_formatter(self.kwdict)))
except Exception as exc: except Exception as exc:
raise exception.FormatError(exc, "filename") raise exception.FormatError(exc, "filename")