mirror of
https://github.com/blackjack4494/yt-dlc.git
synced 2024-09-12 04:22:35 +02:00
Merge pull request #129 from jbruchon/master
Switch from binary search tree to Python sets
This commit is contained in:
commit
486ad2cd50
@ -114,54 +114,6 @@ if compat_os_name == 'nt':
|
|||||||
import ctypes
|
import ctypes
|
||||||
|
|
||||||
|
|
||||||
class ArchiveTree(object):
|
|
||||||
"""Binary search tree for download archive entries"""
|
|
||||||
def __init__(self, line):
|
|
||||||
self.left = None
|
|
||||||
self.right = None
|
|
||||||
self.line = line
|
|
||||||
|
|
||||||
# Tree insertion
|
|
||||||
def at_insert(self, line):
|
|
||||||
cur = self
|
|
||||||
while True:
|
|
||||||
if cur.line:
|
|
||||||
if line < cur.line:
|
|
||||||
if cur.left is None:
|
|
||||||
cur.left = ArchiveTree(line)
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
cur = cur.left
|
|
||||||
continue
|
|
||||||
elif line > cur.line:
|
|
||||||
if cur.right is None:
|
|
||||||
cur.right = ArchiveTree(line)
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
cur = cur.right
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
# Duplicate line found
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
cur.line = line
|
|
||||||
return
|
|
||||||
|
|
||||||
def at_exist(self, line):
|
|
||||||
if self.line is None:
|
|
||||||
return False
|
|
||||||
if line < self.line:
|
|
||||||
if self.left is None:
|
|
||||||
return False
|
|
||||||
return self.left.at_exist(line)
|
|
||||||
elif line > self.line:
|
|
||||||
if self.right is None:
|
|
||||||
return False
|
|
||||||
return self.right.at_exist(line)
|
|
||||||
else:
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
class YoutubeDL(object):
|
class YoutubeDL(object):
|
||||||
"""YoutubeDL class.
|
"""YoutubeDL class.
|
||||||
|
|
||||||
@ -407,38 +359,21 @@ class YoutubeDL(object):
|
|||||||
}
|
}
|
||||||
self.params.update(params)
|
self.params.update(params)
|
||||||
self.cache = Cache(self)
|
self.cache = Cache(self)
|
||||||
self.archive = ArchiveTree(None)
|
self.archive = set()
|
||||||
|
|
||||||
"""Preload the archive, if any is specified"""
|
"""Preload the archive, if any is specified"""
|
||||||
def preload_download_archive(self):
|
def preload_download_archive(self):
|
||||||
lines = []
|
|
||||||
fn = self.params.get('download_archive')
|
fn = self.params.get('download_archive')
|
||||||
if fn is None:
|
if fn is None:
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
with locked_file(fn, 'r', encoding='utf-8') as archive_file:
|
with locked_file(fn, 'r', encoding='utf-8') as archive_file:
|
||||||
for line in archive_file:
|
for line in archive_file:
|
||||||
lines.append(line.strip())
|
self.archive.add(line.strip())
|
||||||
except IOError as ioe:
|
except IOError as ioe:
|
||||||
if ioe.errno != errno.ENOENT:
|
if ioe.errno != errno.ENOENT:
|
||||||
raise
|
raise
|
||||||
lmax = len(lines)
|
|
||||||
if lmax > 10:
|
|
||||||
pos = 0
|
|
||||||
while pos < lmax:
|
|
||||||
if lmax - pos <= 2:
|
|
||||||
break
|
|
||||||
target = random.randrange(pos + 1, lmax - 1)
|
|
||||||
# Swap line at pos with randomly chosen target
|
|
||||||
temp = lines[pos]
|
|
||||||
lines[pos] = lines[target]
|
|
||||||
lines[target] = temp
|
|
||||||
pos += 1
|
|
||||||
elif lmax < 1:
|
|
||||||
# No lines were loaded
|
|
||||||
return False
|
return False
|
||||||
for x in lines:
|
|
||||||
self.archive.at_insert(x)
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def check_deprecated(param, option, suggestion):
|
def check_deprecated(param, option, suggestion):
|
||||||
@ -2228,7 +2163,7 @@ class YoutubeDL(object):
|
|||||||
if not vid_id:
|
if not vid_id:
|
||||||
return False # Incomplete video information
|
return False # Incomplete video information
|
||||||
|
|
||||||
return self.archive.at_exist(vid_id)
|
return vid_id in self.archive
|
||||||
|
|
||||||
def record_download_archive(self, info_dict):
|
def record_download_archive(self, info_dict):
|
||||||
fn = self.params.get('download_archive')
|
fn = self.params.get('download_archive')
|
||||||
@ -2238,7 +2173,7 @@ class YoutubeDL(object):
|
|||||||
assert vid_id
|
assert vid_id
|
||||||
with locked_file(fn, 'a', encoding='utf-8') as archive_file:
|
with locked_file(fn, 'a', encoding='utf-8') as archive_file:
|
||||||
archive_file.write(vid_id + '\n')
|
archive_file.write(vid_id + '\n')
|
||||||
self.archive.at_insert(vid_id)
|
self.archive.add(vid_id)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def format_resolution(format, default='unknown'):
|
def format_resolution(format, default='unknown'):
|
||||||
|
Loading…
Reference in New Issue
Block a user