Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-26 04:32:51 +01:00
Code Issues Releases Wiki Activity

optimize directory path generation

Browse Source 
- use str.join() instead of os.path.join()
  (less "features", but 10x as fast)
- cache directory formatters
- detect and optimize field access for 1-element format strings
This commit is contained in:
Mike Fährmann 2019-08-19 15:56:20 +02:00
parent 51d10783fc
commit e77a656437
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
1 changed files with 35 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
gallery_dl/util.py
View File

@ -391,10 +391,18 @@ class Formatter():
if field_name:
self.fields.append((
len(self.result),
self._field_access(field_name, format_spec, conversion)
self._field_access(field_name, format_spec, conversion),
))
self.result.append("")
if len(self.result) == 1:
if self.fields:
self.format_map = self.fields[0][1]
else:
self.format_map = lambda _: format_string
del self.result
del self.fields
def format_map(self, kwargs):
"""Apply 'kwargs' to the initial format_string and return its result"""
for index, func in self.fields:
@ -512,17 +520,24 @@ class Formatter():
class PathFormat():
def __init__(self, extractor):
self.filename_fmt = extractor.config(
"filename", extractor.filename_fmt)
self.directory_fmt = extractor.config(
"directory", extractor.directory_fmt)
self.kwdefault = extractor.config("keywords-default")
filename_fmt = extractor.config("filename", extractor.filename_fmt)
directory_fmt = extractor.config("directory", extractor.directory_fmt)
kwdefault = extractor.config("keywords-default")
try:
self.formatter = Formatter(self.filename_fmt, self.kwdefault)
self.filename_formatter = Formatter(
filename_fmt, kwdefault).format_map
except Exception as exc:
raise exception.FormatError(exc, "filename")
try:
self.directory_formatters = [
Formatter(dirfmt, kwdefault).format_map
for dirfmt in directory_fmt
]
except Exception as exc:
raise exception.FormatError(exc, "directory")
self.directory = self.realdirectory = ""
self.filename = ""
self.extension = ""
@ -535,6 +550,8 @@ class PathFormat():
extractor.config("base-directory", (".", "gallery-dl")))
if os.altsep and os.altsep in self.basedirectory:
self.basedirectory = self.basedirectory.replace(os.altsep, os.sep)
if self.basedirectory[-1] != os.sep:
self.basedirectory += os.sep
restrict = extractor.config("path-restrict", "auto")
if restrict == "auto":
@ -592,30 +609,27 @@ class PathFormat():
# Build path segments by applying 'kwdict' to directory format strings
try:
segments = [
self.clean_segment(
Formatter(segment, self.kwdefault)
.format_map(kwdict)
.strip()
)
for segment in self.directory_fmt
self.clean_segment(format_map(kwdict).strip())
for format_map in self.directory_formatters
]
except Exception as exc:
raise exception.FormatError(exc, "directory")
# Join path segements
self.directory = self.clean_path(os.path.join(
self.basedirectory, *segments))
directory = self.clean_path(
self.basedirectory + os.sep.join(segments))
# Remove trailing path separator;
# occurs if the last argument to os.path.join() is an empty string
if self.directory[-1] == os.sep:
self.directory = self.directory[:-1]
# occurs if the last segment is an empty string
if directory[-1] == os.sep:
directory = directory[:-1]
self.directory = directory
# Enable longer-than-260-character paths on Windows
if os.name == "nt":
self.realdirectory = "\\\\?\\" + os.path.abspath(self.directory)
self.realdirectory = "\\\\?\\" + os.path.abspath(directory)
else:
self.realdirectory = self.directory
self.realdirectory = directory
# Create directory tree
os.makedirs(self.realdirectory, exist_ok=True)
@ -651,7 +665,7 @@ class PathFormat():
# Apply 'kwdict' to filename format string
try:
self.filename = self.clean_path(self.clean_segment(
self.formatter.format_map(self.kwdict)))
self.filename_formatter(self.kwdict)))
except Exception as exc:
raise exception.FormatError(exc, "filename")