kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Option to choose different downloader for different protocols

Browse source 
* Renamed `--external-downloader-args` to `--downloader-args`
* Added `native` as an option for the downloader
* Use similar syntax to `--downloader-args` etc. Eg: `--downloader dash:native --downloader aria2c`
* Deprecated `--hls-prefer-native` and `--hls-prefer-ffmpeg` since the same can now be done with `--downloader "m3u8:native"` and `m3u8:ffmpeg` respectively
* Split `frag_urls` protocol into `m3u8_frag_urls` and `dash_frag_urls`
* Standardize shortening of protocol names with `downloader.shorten_protocol_name`
This commit is contained in:
pukkandan 2021-04-10 20:38:33 +05:30
parent d818eb7473
commit 52a8a1e1b9
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
7 changed files with 95 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
README.md
View file

@ -337,10 +337,6 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
--playlist-random Download playlist videos in random order
--xattr-set-filesize Set file xattribute ytdl.filesize with
expected file size
--hls-prefer-native Use the native HLS downloader instead of
ffmpeg
--hls-prefer-ffmpeg Use ffmpeg instead of the native HLS
downloader
--hls-use-mpegts Use the mpegts container for HLS videos;
allowing some players to play the video
while downloading, and reducing the chance
@ -350,10 +346,19 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
--no-hls-use-mpegts Do not use the mpegts container for HLS
videos. This is default when not
downloading live streams
--external-downloader NAME Name or path of the external downloader to
use. Currently supports aria2c, avconv,
axel, curl, ffmpeg, httpie, wget
(Recommended: aria2c)
--downloader [PROTO:]NAME Name or path of the external downloader to
use (optionally) prefixed by the protocols
(http, ftp, m3u8, dash, rstp, rtmp, mms) to
use it for. Currently supports native,
aria2c, avconv, axel, curl, ffmpeg, httpie,
wget (Recommended: aria2c). You can use
this option multiple times to set different
downloaders for different protocols. For
example, --downloader aria2c --downloader
"dash,m3u8:native" will use aria2c for
http/ftp downloads, and the native
downloader for dash/m3u8 downloads
(Alias: --external-downloader)
--downloader-args NAME:ARGS Give these arguments to the external
downloader. Specify the downloader name and
the arguments separated by a colon ":". You
@ -1244,6 +1249,8 @@ These are all the deprecated options and the current alternative to achieve the
--metadata-from-title FORMAT --parse-metadata "%(title)s:FORMAT"
--prefer-avconv avconv is no longer officially supported (Alias: --no-prefer-ffmpeg)
--prefer-ffmpeg Default (Alias: --no-prefer-avconv)
--hls-prefer-native --downloader "m3u8:native"
--hls-prefer-ffmpeg --downloader "m3u8:ffmpeg"
--avconv-location avconv is no longer officially supported
-C, --call-home Not implemented
--no-call-home Default

24
yt_dlp/YoutubeDL.py
View file

@ -111,9 +111,17 @@ from .utils import (
process_communicate_or_kill,
)
from .cache import Cache
from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER, _PLUGIN_CLASSES
from .extractor import (
gen_extractor_classes,
get_info_extractor,
_LAZY_LOADER,
_PLUGIN_CLASSES
)
from .extractor.openload import PhantomJSwrapper
from .downloader import get_suitable_downloader
from .downloader import (
get_suitable_downloader,
shorten_protocol_name
)
from .downloader.rtmp import rtmpdump_version
from .postprocessor import (
FFmpegFixupM3u8PP,
@ -359,9 +367,13 @@ class YoutubeDL(object):
geo_bypass_country
The following options determine which downloader is picked:
external_downloader: Executable of the external downloader to call.
None or unset for standard (built-in) downloader.
hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
external_downloader: A dictionary of protocol keys and the executable of the
external downloader to use for it. The allowed protocols
are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
Set the value to 'native' to use the native downloader
hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
or {'m3u8': 'ffmpeg'} instead.
Use the native HLS downloader instead of ffmpeg/avconv
if True, otherwise use ffmpeg/avconv if False, otherwise
use downloader suggested by extractor if None.
@ -2776,7 +2788,7 @@ class YoutubeDL(object):
'|',
format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
format_field(f, 'tbr', '%4dk'),
f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''),
shorten_protocol_name(f.get('protocol', '').replace("native", "n")),
'|',
format_field(f, 'vcodec', default='unknown').replace('none', ''),
format_field(f, 'vbr', '%4dk'),

33
yt_dlp/downloader/__init__.py
View file

@ -1,5 +1,6 @@
from __future__ import unicode_literals
from ..compat import compat_str
from ..utils import (
determine_protocol,
)
@ -42,6 +43,23 @@ PROTOCOL_MAP = {
}
def shorten_protocol_name(proto, simplify=False):
short_protocol_names = {
'm3u8_native': 'm3u8_n',
'http_dash_segments': 'dash',
'niconico_dmc': 'dmc',
}
if simplify:
short_protocol_names.update({
'https': 'http',
'ftps': 'ftp',
'm3u8_native': 'm3u8',
'm3u8_frag_urls': 'm3u8',
'dash_frag_urls': 'dash',
})
return short_protocol_names.get(proto, proto)
def get_suitable_downloader(info_dict, params={}, default=HttpFD):
"""Get the downloader class that can handle the info dict."""
protocol = determine_protocol(info_dict)
@ -50,8 +68,14 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
# if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
# return FFmpegFD
external_downloader = params.get('external_downloader')
if external_downloader is not None:
downloaders = params.get('external_downloader')
external_downloader = (
downloaders if isinstance(downloaders, compat_str)
else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default')))
if external_downloader and external_downloader.lower() == 'native':
external_downloader = 'native'
if external_downloader not in (None, 'native'):
ed = get_external_downloader(external_downloader)
if ed.can_download(info_dict, external_downloader):
return ed
@ -59,6 +83,8 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
if protocol.startswith('m3u8'):
if info_dict.get('is_live'):
return FFmpegFD
elif external_downloader == 'native':
return HlsFD
elif _get_real_downloader(info_dict, 'frag_urls', params, None):
return HlsFD
elif params.get('hls_prefer_native') is True:
@ -70,6 +96,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
__all__ = [
'get_suitable_downloader',
'FileDownloader',
'get_suitable_downloader',
'shorten_protocol_name',
]

4
yt_dlp/downloader/dash.py
View file

@ -20,7 +20,7 @@ from ..utils import (
class DashSegmentsFD(FragmentFD):
"""
Download segments in a DASH manifest. External downloaders can take over
the fragment downloads by supporting the 'frag_urls' protocol
the fragment downloads by supporting the 'dash_frag_urls' protocol
"""
FD_NAME = 'dashsegments'
@ -30,7 +30,7 @@ class DashSegmentsFD(FragmentFD):
fragments = info_dict['fragments'][:1] if self.params.get(
'test', False) else info_dict['fragments']
real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None)
ctx = {
'filename': filename,

19
yt_dlp/downloader/external.py
View file

@ -81,11 +81,15 @@ class ExternalFD(FileDownloader):
@property
def exe(self):
return self.params.get('external_downloader')
return self.get_basename()
@classmethod
def available(cls, path=None):
return check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
path = check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
if path:
cls.exe = path
return path
return False
@classmethod
def supports(cls, info_dict):
@ -259,7 +263,7 @@ class WgetFD(ExternalFD):
class Aria2cFD(ExternalFD):
AVAILABLE_OPT = '-v'
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'frag_urls')
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls')
@staticmethod
def supports_manifest(manifest):
@ -310,9 +314,11 @@ class Aria2cFD(ExternalFD):
class HttpieFD(ExternalFD):
AVAILABLE_OPT = '--version'
@classmethod
def available(cls, path=None):
return check_executable(path or 'http', ['--version'])
return ExternalFD.available(cls, path or 'http')
def _make_cmd(self, tmpfilename, info_dict):
cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
@ -327,7 +333,8 @@ class FFmpegFD(ExternalFD):
SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
@classmethod
def available(cls, path=None): # path is ignored for ffmpeg
def available(cls, path=None):
# TODO: Fix path for ffmpeg
return FFmpegPostProcessor().available
def _call_downloader(self, tmpfilename, info_dict):
@ -484,4 +491,4 @@ def get_external_downloader(external_downloader):
downloader . """
# Drop .exe extension on Windows
bn = os.path.splitext(os.path.basename(external_downloader))[0]
return _BY_NAME[bn]
return _BY_NAME.get(bn)

4
yt_dlp/downloader/hls.py
View file

@ -32,7 +32,7 @@ from ..utils import (
class HlsFD(FragmentFD):
"""
Download segments in a m3u8 manifest. External downloaders can take over
the fragment downloads by supporting the 'frag_urls' protocol and
the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
re-defining 'supports_manifest' function
"""
@ -95,7 +95,7 @@ class HlsFD(FragmentFD):
# fd.add_progress_hook(ph)
return fd.real_download(filename, info_dict)
real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
if real_downloader and not real_downloader.supports_manifest(s):
real_downloader = None
if real_downloader:

21
yt_dlp/options.py
View file

@ -639,11 +639,11 @@ def parseOpts(overrideArguments=None):
downloader.add_option(
'--hls-prefer-native',
dest='hls_prefer_native', action='store_true', default=None,
help='Use the native HLS downloader instead of ffmpeg')
help=optparse.SUPPRESS_HELP)
downloader.add_option(
'--hls-prefer-ffmpeg',
dest='hls_prefer_native', action='store_false', default=None,
help='Use ffmpeg instead of the native HLS downloader')
help=optparse.SUPPRESS_HELP)
downloader.add_option(
'--hls-use-mpegts',
dest='hls_use_mpegts', action='store_true', default=None,
@ -659,11 +659,20 @@ def parseOpts(overrideArguments=None):
'Do not use the mpegts container for HLS videos. '
'This is default when not downloading live streams'))
downloader.add_option(
'--external-downloader',
dest='external_downloader', metavar='NAME',
'--downloader', '--external-downloader',
dest='external_downloader', metavar='[PROTO:]NAME', default={}, type='str',
action='callback', callback=_dict_from_multiple_values_options_callback,
callback_kwargs={
'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
'default_key': 'default', 'process': lambda x: x.strip()},
help=(
'Name or path of the external downloader to use. '
'Currently supports %s (Recommended: aria2c)' % ', '.join(list_external_downloaders())))
'Name or path of the external downloader to use (optionally) prefixed by '
'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
'Currently supports native, %s (Recommended: aria2c). '
'You can use this option multiple times to set different downloaders for different protocols. '
'For example, --downloader aria2c --downloader "dash,m3u8:native" will use '
'aria2c for http/ftp downloads, and the native downloader for dash/m3u8 downloads '
'(Alias: --external-downloader)' % ', '.join(list_external_downloaders())))
downloader.add_option(
'--downloader-args', '--external-downloader-args',
metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',