kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[downloader] Fix slow progress hooks

Browse source 
Closes #1301
This commit is contained in:
pukkandan 2021-10-16 18:31:00 +05:30
parent 48ee10ee8a
commit 03b4de722a
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
5 changed files with 23 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
yt_dlp/YoutubeDL.py
View file

@ -950,13 +950,18 @@ class YoutubeDL(object):
except ValueError as err:
return err
@staticmethod
def _copy_infodict(info_dict):
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
return info_dict
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
""" Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
info_dict = self._copy_infodict(info_dict)
info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
formatSeconds(info_dict['duration'], '-' if sanitize else ':')
if info_dict.get('duration', None) is not None
@ -2265,7 +2270,7 @@ class YoutubeDL(object):
formats_dict[format_id].append(format)
# Make sure all formats have unique format_id
common_exts = set(ext for exts in self._format_selection_exts.values() for ext in exts)
common_exts = set(itertools.chain(*self._format_selection_exts.values()))
for format_id, ambiguous_formats in formats_dict.items():
ambigious_id = len(ambiguous_formats) > 1
for i, format in enumerate(ambiguous_formats):
@ -2523,7 +2528,8 @@ class YoutubeDL(object):
fd.add_progress_hook(ph)
urls = '", "'.join([f['url'] for f in info.get('requested_formats', [])] or [info['url']])
self.write_debug('Invoking downloader on "%s"' % urls)
new_info = dict(info)
new_info = copy.deepcopy(self._copy_infodict(info))
if new_info.get('http_headers') is None:
new_info['http_headers'] = self._calc_headers(new_info)
return fd.download(name, new_info, subtitle)

5
yt_dlp/downloader/common.py
View file

@ -405,13 +405,10 @@ class FileDownloader(object):
def _hook_progress(self, status, info_dict):
if not self._progress_hooks:
return
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
status['info_dict'] = info_dict
# youtube-dl passes the same status object to all the hooks.
# Some third party scripts seems to be relying on this.
# So keep this behavior if possible
status['info_dict'] = copy.deepcopy(info_dict)
for ph in self._progress_hooks:
ph(status)

5
yt_dlp/downloader/dash.py
View file

@ -55,9 +55,8 @@ class DashSegmentsFD(FragmentFD):
if real_downloader:
self.to_screen(
'[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
info_copy = info_dict.copy()
info_copy['fragments'] = fragments_to_download
info_dict['fragments'] = fragments_to_download
fd = real_downloader(self.ydl, self.params)
return fd.real_download(filename, info_copy)
return fd.real_download(filename, info_dict)
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)

5
yt_dlp/downloader/hls.py
View file

@ -245,13 +245,12 @@ class HlsFD(FragmentFD):
fragments = [fragments[0] if fragments else None]
if real_downloader:
info_copy = info_dict.copy()
info_copy['fragments'] = fragments
info_dict['fragments'] = fragments
fd = real_downloader(self.ydl, self.params)
# TODO: Make progress updates work without hooking twice
# for ph in self._progress_hooks:
# fd.add_progress_hook(ph)
return fd.real_download(filename, info_copy)
return fd.real_download(filename, info_dict)
if is_webvtt:
def pack_fragment(frag_content, frag_index):

13
yt_dlp/postprocessor/common.py
View file

@ -17,11 +17,12 @@ class PostProcessorMetaClass(type):
def run_wrapper(func):
@functools.wraps(func)
def run(self, info, *args, **kwargs):
self._hook_progress({'status': 'started'}, info)
info_copy = copy.deepcopy(self._copy_infodict(info))
self._hook_progress({'status': 'started'}, info_copy)
ret = func(self, info, *args, **kwargs)
if ret is not None:
_, info = ret
self._hook_progress({'status': 'finished'}, info)
self._hook_progress({'status': 'finished'}, info_copy)
return ret
return run
@ -93,6 +94,9 @@ class PostProcessor(metaclass=PostProcessorMetaClass):
for ph in getattr(downloader, '_postprocessor_hooks', []):
self.add_progress_hook(ph)
def _copy_infodict(self, info_dict):
return getattr(self._downloader, '_copy_infodict', dict)(info_dict)
@staticmethod
def _restrict_to(*, video=True, audio=True, images=True):
allowed = {'video': video, 'audio': audio, 'images': images}
@ -142,11 +146,8 @@ class PostProcessor(metaclass=PostProcessorMetaClass):
def _hook_progress(self, status, info_dict):
if not self._progress_hooks:
return
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
status.update({
'info_dict': copy.deepcopy(info_dict),
'info_dict': info_dict,
'postprocessor': self.pp_key(),
})
for ph in self._progress_hooks: