kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[cleanup] Refactor some code

Browse source
This commit is contained in:
pukkandan 2021-07-31 16:21:01 +05:30
parent d74a58a186
commit dbf5416a20
9 changed files with 67 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
yt_dlp/YoutubeDL.py
View file

@ -126,6 +126,7 @@ from .extractor import (
)
from .extractor.openload import PhantomJSwrapper
from .downloader import (
FFmpegFD,
get_suitable_downloader,
shorten_protocol_name
)
@ -2690,20 +2691,15 @@ class YoutubeDL(object):
info_dict['__real_download'] = False
_protocols = set(determine_protocol(f) for f in requested_formats)
if len(_protocols) == 1:
if len(_protocols) == 1: # All requested formats have same protocol
info_dict['protocol'] = _protocols.pop()
directly_mergable = (
'no-direct-merge' not in self.params.get('compat_opts', [])
and info_dict.get('protocol') is not None # All requested formats have same protocol
and not self.params.get('allow_unplayable_formats')
and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD')
if directly_mergable:
info_dict['url'] = requested_formats[0]['url']
# Treat it as a single download
dl_filename = existing_file(full_filename, temp_filename)
if dl_filename is None:
success, real_download = self.dl(temp_filename, info_dict)
info_dict['__real_download'] = real_download
directly_mergable = FFmpegFD.can_merge_formats(info_dict)
if dl_filename is not None:
pass
elif (directly_mergable and get_suitable_downloader(info_dict, self.params) == FFmpegFD):
info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
success, real_download = self.dl(temp_filename, info_dict)
info_dict['__real_download'] = real_download
else:
downloaded = []
merger = FFmpegMergerPP(self)
@ -2717,28 +2713,25 @@ class YoutubeDL(object):
'You have requested merging of multiple formats but ffmpeg is not installed. '
'The formats won\'t be merged.')
if dl_filename is None:
for f in requested_formats:
new_info = dict(info_dict)
del new_info['requested_formats']
new_info.update(f)
fname = prepend_extension(
self.prepare_filename(new_info, 'temp'),
'f%s' % f['format_id'], new_info['ext'])
if not self._ensure_dir_exists(fname):
return
downloaded.append(fname)
partial_success, real_download = self.dl(fname, new_info)
info_dict['__real_download'] = info_dict['__real_download'] or real_download
success = success and partial_success
if merger.available and not self.params.get('allow_unplayable_formats'):
info_dict['__postprocessors'].append(merger)
info_dict['__files_to_merge'] = downloaded
# Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True
else:
for file in downloaded:
files_to_move[file] = None
for f in requested_formats:
new_info = dict(info_dict)
del new_info['requested_formats']
new_info.update(f)
fname = prepend_extension(temp_filename, 'f%s' % f['format_id'], new_info['ext'])
if not self._ensure_dir_exists(fname):
return
downloaded.append(fname)
partial_success, real_download = self.dl(fname, new_info)
info_dict['__real_download'] = info_dict['__real_download'] or real_download
success = success and partial_success
if merger.available and not self.params.get('allow_unplayable_formats'):
info_dict['__postprocessors'].append(merger)
info_dict['__files_to_merge'] = downloaded
# Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True
else:
for file in downloaded:
files_to_move[file] = None
else:
# Just a single file
dl_filename = existing_file(full_filename, temp_filename)

17
yt_dlp/downloader/__init__.py
View file

@ -3,17 +3,19 @@ from __future__ import unicode_literals
from ..compat import compat_str
from ..utils import (
determine_protocol,
NO_DEFAULT
)
def _get_real_downloader(info_dict, protocol=None, *args, **kwargs):
def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None):
info_dict['protocol'] = determine_protocol(info_dict)
info_copy = info_dict.copy()
if protocol:
info_copy['protocol'] = protocol
return get_suitable_downloader(info_copy, *args, **kwargs)
return _get_suitable_downloader(info_copy, params, default)
# Some of these require _get_real_downloader
# Some of these require get_suitable_downloader
from .common import FileDownloader
from .dash import DashSegmentsFD
from .f4m import F4mFD
@ -69,14 +71,15 @@ def shorten_protocol_name(proto, simplify=False):
return short_protocol_names.get(proto, proto)
def get_suitable_downloader(info_dict, params={}, default=HttpFD):
def _get_suitable_downloader(info_dict, params, default):
"""Get the downloader class that can handle the info dict."""
protocol = determine_protocol(info_dict)
info_dict['protocol'] = protocol
if default is NO_DEFAULT:
default = HttpFD
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
# return FFmpegFD
protocol = info_dict['protocol']
downloaders = params.get('external_downloader')
external_downloader = (
downloaders if isinstance(downloaders, compat_str) or downloaders is None
@ -94,7 +97,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
return FFmpegFD
elif external_downloader == 'native':
return HlsFD
elif _get_real_downloader(info_dict, 'm3u8_frag_urls', params, None):
elif get_suitable_downloader(info_dict, params, None, protocol='m3u8_frag_urls'):
return HlsFD
elif params.get('hls_prefer_native') is True:
return HlsFD

10
yt_dlp/downloader/dash.py
View file

@ -1,6 +1,6 @@
from __future__ import unicode_literals
from ..downloader import _get_real_downloader
from ..downloader import get_suitable_downloader
from .fragment import FragmentFD
from ..utils import urljoin
@ -15,11 +15,14 @@ class DashSegmentsFD(FragmentFD):
FD_NAME = 'dashsegments'
def real_download(self, filename, info_dict):
if info_dict.get('is_live'):
self.report_error('Live DASH videos are not supported')
fragment_base_url = info_dict.get('fragment_base_url')
fragments = info_dict['fragments'][:1] if self.params.get(
'test', False) else info_dict['fragments']
real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None)
real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='dash_frag_urls')
ctx = {
'filename': filename,
@ -54,9 +57,6 @@ class DashSegmentsFD(FragmentFD):
info_copy = info_dict.copy()
info_copy['fragments'] = fragments_to_download
fd = real_downloader(self.ydl, self.params)
# TODO: Make progress updates work without hooking twice
# for ph in self._progress_hooks:
# fd.add_progress_hook(ph)
return fd.real_download(filename, info_copy)
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)

10
yt_dlp/downloader/external.py
View file

@ -345,12 +345,22 @@ class FFmpegFD(ExternalFD):
@classmethod
def available(cls, path=None):
# TODO: Fix path for ffmpeg
# Fixme: This may be wrong when --ffmpeg-location is used
return FFmpegPostProcessor().available
def on_process_started(self, proc, stdin):
""" Override this in subclasses """
pass
@classmethod
def can_merge_formats(cls, info_dict, params={}):
return (
info_dict.get('requested_formats')
and info_dict.get('protocol')
and not params.get('allow_unplayable_formats')
and 'no-direct-merge' not in params.get('compat_opts', [])
and cls.can_download(info_dict))
def _call_downloader(self, tmpfilename, info_dict):
urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
ffpp = FFmpegPostProcessor(downloader=self)

7
yt_dlp/downloader/hls.py
View file

@ -4,7 +4,7 @@ import re
import io
import binascii
from ..downloader import _get_real_downloader
from ..downloader import get_suitable_downloader
from .fragment import FragmentFD, can_decrypt_frag
from .external import FFmpegFD
@ -80,16 +80,13 @@ class HlsFD(FragmentFD):
fd = FFmpegFD(self.ydl, self.params)
self.report_warning(
'%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename()))
# TODO: Make progress updates work without hooking twice
# for ph in self._progress_hooks:
# fd.add_progress_hook(ph)
return fd.real_download(filename, info_dict)
is_webvtt = info_dict['ext'] == 'vtt'
if is_webvtt:
real_downloader = None # Packing the fragments is not currently supported for external downloader
else:
real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='m3u8_frag_urls')
if real_downloader and not real_downloader.supports_manifest(s):
real_downloader = None
if real_downloader:

4
yt_dlp/downloader/niconico.py
View file

@ -4,7 +4,7 @@ from __future__ import unicode_literals
import threading
from .common import FileDownloader
from ..downloader import _get_real_downloader
from ..downloader import get_suitable_downloader
from ..extractor.niconico import NiconicoIE
from ..compat import compat_urllib_request
@ -20,7 +20,7 @@ class NiconicoDmcFD(FileDownloader):
ie = NiconicoIE(self.ydl)
info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
fd = _get_real_downloader(info_dict, params=self.params)(self.ydl, self.params)
fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
success = download_complete = False
timer = [None]

4
yt_dlp/extractor/common.py
View file

@ -1298,7 +1298,7 @@ class InfoExtractor(object):
# JSON-LD may be malformed and thus `fatal` should be respected.
# At the same time `default` may be passed that assumes `fatal=False`
# for _search_regex. Let's simulate the same behavior here as well.
fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
fatal = kwargs.get('fatal', True) if default is NO_DEFAULT else False
json_ld = []
for mobj in json_ld_list:
json_ld_item = self._parse_json(
@ -1522,7 +1522,7 @@ class InfoExtractor(object):
'size': {'type': 'combined', 'same_limit': True, 'field': ('filesize', 'fs_approx')},
'ext': {'type': 'combined', 'field': ('vext', 'aext')},
'res': {'type': 'multiple', 'field': ('height', 'width'),
'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
# Most of these exist only for compatibility reasons
'dimension': {'type': 'alias', 'field': 'res'},

17
yt_dlp/options.py
View file

@ -706,9 +706,8 @@ def parseOpts(overrideArguments=None):
callback_kwargs={
'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
'default_key': 'default',
'process': lambda x: x.strip()
},
help=(
'process': str.strip
}, help=(
'Name or path of the external downloader to use (optionally) prefixed by '
'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
'Currently supports native, %s (Recommended: aria2c). '
@ -724,8 +723,7 @@ def parseOpts(overrideArguments=None):
'allowed_keys': '|'.join(list_external_downloaders()),
'default_key': 'default',
'process': compat_shlex_split
},
help=(
}, help=(
'Give these arguments to the external downloader. '
'Specify the downloader name and the arguments separated by a colon ":". '
'You can use this option multiple times to give different arguments to different downloaders '
@ -944,8 +942,7 @@ def parseOpts(overrideArguments=None):
callback_kwargs={
'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()),
'default_key': 'default'
},
help='Output filename template; see "OUTPUT TEMPLATE" for details')
}, help='Output filename template; see "OUTPUT TEMPLATE" for details')
filesystem.add_option(
'--output-na-placeholder',
dest='outtmpl_na_placeholder', metavar='TEXT', default='NA',
@ -1191,8 +1188,7 @@ def parseOpts(overrideArguments=None):
'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat',
'process': compat_shlex_split,
'multiple_keys': False
},
help=(
}, help=(
'Give these arguments to the postprocessors. '
'Specify the postprocessor/executable name and the arguments separated by a colon ":" '
'to give the argument to the specified postprocessor/executable. Supported PP are: '
@ -1385,8 +1381,7 @@ def parseOpts(overrideArguments=None):
'multiple_keys': False,
'process': lambda val: dict(
_extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';'))
},
help=(
}, help=(
'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. '
'You can use this option multiple times to give arguments for different extractors'))
extractor.add_option(

4
yt_dlp/utils.py
View file

@ -4544,7 +4544,7 @@ def parse_codecs(codecs_str):
if not codecs_str:
return {}
split_codecs = list(filter(None, map(
lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
str.strip, codecs_str.strip().strip(',').split(','))))
vcodec, acodec = None, None
for full_codec in split_codecs:
codec = full_codec.split('.')[0]
@ -6246,7 +6246,7 @@ def traverse_obj(
# TODO: Write tests
'''
if not casesense:
_lower = lambda k: k.lower() if isinstance(k, str) else k
_lower = lambda k: (k.lower() if isinstance(k, str) else k)
path_list = (map(_lower, variadic(path)) for path in path_list)
def _traverse_obj(obj, path, _current_depth=0):