kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #129 from jbruchon/master

Browse source 
Switch from binary search tree to Python sets
This commit is contained in:
Tom-Oliver Heidel 2020-09-20 12:14:03 +02:00 committed by GitHub
commit 486ad2cd50
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
youtube_dlc/YoutubeDL.py
View file

@ -114,54 +114,6 @@ if compat_os_name == 'nt':
import ctypes
class ArchiveTree(object):
"""Binary search tree for download archive entries"""
def __init__(self, line):
self.left = None
self.right = None
self.line = line
# Tree insertion
def at_insert(self, line):
cur = self
while True:
if cur.line:
if line < cur.line:
if cur.left is None:
cur.left = ArchiveTree(line)
return
else:
cur = cur.left
continue
elif line > cur.line:
if cur.right is None:
cur.right = ArchiveTree(line)
return
else:
cur = cur.right
continue
else:
# Duplicate line found
return
else:
cur.line = line
return
def at_exist(self, line):
if self.line is None:
return False
if line < self.line:
if self.left is None:
return False
return self.left.at_exist(line)
elif line > self.line:
if self.right is None:
return False
return self.right.at_exist(line)
else:
return True
class YoutubeDL(object):
"""YoutubeDL class.
@ -407,38 +359,21 @@ class YoutubeDL(object):
}
self.params.update(params)
self.cache = Cache(self)
self.archive = ArchiveTree(None)
self.archive = set()
"""Preload the archive, if any is specified"""
def preload_download_archive(self):
lines = []
fn = self.params.get('download_archive')
if fn is None:
return False
try:
with locked_file(fn, 'r', encoding='utf-8') as archive_file:
for line in archive_file:
lines.append(line.strip())
self.archive.add(line.strip())
except IOError as ioe:
if ioe.errno != errno.ENOENT:
raise
lmax = len(lines)
if lmax > 10:
pos = 0
while pos < lmax:
if lmax - pos <= 2:
break
target = random.randrange(pos + 1, lmax - 1)
# Swap line at pos with randomly chosen target
temp = lines[pos]
lines[pos] = lines[target]
lines[target] = temp
pos += 1
elif lmax < 1:
# No lines were loaded
return False
for x in lines:
self.archive.at_insert(x)
return True
def check_deprecated(param, option, suggestion):
@ -2228,7 +2163,7 @@ class YoutubeDL(object):
if not vid_id:
return False # Incomplete video information
return self.archive.at_exist(vid_id)
return vid_id in self.archive
def record_download_archive(self, info_dict):
fn = self.params.get('download_archive')
@ -2238,7 +2173,7 @@ class YoutubeDL(object):
assert vid_id
with locked_file(fn, 'a', encoding='utf-8') as archive_file:
archive_file.write(vid_id + '\n')
self.archive.at_insert(vid_id)
self.archive.add(vid_id)
@staticmethod
def format_resolution(format, default='unknown'):