From 23326151c45b632c3d5948bd018e80abb370e676 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 19 May 2022 20:00:31 +0530 Subject: [PATCH] Add option --retry-sleep (#3059) Closes #2852 --- yt_dlp/YoutubeDL.py | 3 +++ yt_dlp/__init__.py | 23 +++++++++++++++++++++++ yt_dlp/downloader/common.py | 18 ++++++++++++++++-- yt_dlp/downloader/external.py | 1 + yt_dlp/downloader/fragment.py | 5 +---- yt_dlp/options.py | 12 ++++++++++++ 6 files changed, 56 insertions(+), 6 deletions(-) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index ba08f6a7d..749cf9402 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -453,6 +453,9 @@ class YoutubeDL: Allowed keys are 'download', 'postprocess', 'download-title' (console title) and 'postprocess-title'. The template is mapped on a dictionary with keys 'progress' and 'info' + retry_sleep_functions: Dictionary of functions that takes the number of attempts + as argument and returns the time to sleep in seconds. + Allowed keys are 'http', 'fragment', 'file_access' The following parameters are not used by YoutubeDL itself, they are used by the downloader (see yt_dlp/downloader/common.py): diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 81b1716df..b2429f5af 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -247,6 +247,28 @@ def validate_options(opts): opts.extractor_retries = parse_retries('extractor', opts.extractor_retries) opts.file_access_retries = parse_retries('file access', opts.file_access_retries) + # Retry sleep function + def parse_sleep_func(expr): + NUMBER_RE = r'\d+(?:\.\d+)?' + op, start, limit, step, *_ = tuple(re.fullmatch( + rf'(?:(linear|exp)=)?({NUMBER_RE})(?::({NUMBER_RE}))?(?::({NUMBER_RE}))?', + expr.strip()).groups()) + (None, None) + + if op == 'exp': + return lambda n: min(float(start) * (float(step or 2) ** n), float(limit or 'inf')) + else: + default_step = start if op or limit else 0 + return lambda n: min(float(start) + float(step or default_step) * n, float(limit or 'inf')) + + for key, expr in opts.retry_sleep.items(): + if not expr: + del opts.retry_sleep[key] + continue + try: + opts.retry_sleep[key] = parse_sleep_func(expr) + except AttributeError as e: + raise ValueError(f'invalid {key} retry sleep expression {expr!r}: {e}') + # Bytes def parse_bytes(name, value): if value is None: @@ -694,6 +716,7 @@ def parse_options(argv=None): 'file_access_retries': opts.file_access_retries, 'fragment_retries': opts.fragment_retries, 'extractor_retries': opts.extractor_retries, + 'retry_sleep_functions': opts.retry_sleep, 'skip_unavailable_fragments': opts.skip_unavailable_fragments, 'keep_fragments': opts.keep_fragments, 'concurrent_fragment_downloads': opts.concurrent_fragment_downloads, diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 465b5ef99..0b3383071 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -19,6 +19,7 @@ from ..utils import ( encodeFilename, error_to_compat_str, format_bytes, + int_or_none, sanitize_open, shell_quote, timeconvert, @@ -64,6 +65,7 @@ class FileDownloader: useful for bypassing bandwidth throttling imposed by a webserver (experimental) progress_template: See YoutubeDL.py + retry_sleep_functions: See YoutubeDL.py Subclasses of this one must re-define the real_download method. """ @@ -98,6 +100,8 @@ class FileDownloader: def to_screen(self, *args, **kargs): self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs) + __to_screen = to_screen + @property def FD_NAME(self): return re.sub(r'(? fragment_retries: if not skip_unavailable_fragments: self.report_error('Giving up after %s fragment retries' % fragment_retries) diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index 4655f067f..410c8c1a4 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -25,10 +25,6 @@ class HttpQuietDownloader(HttpFD): console_title = to_screen - def report_retry(self, err, count, retries): - super().to_screen( - f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...') - class FragmentFD(FileDownloader): """ @@ -70,6 +66,7 @@ class FragmentFD(FileDownloader): self.to_screen( '\r[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s) ...' % (error_to_compat_str(err), frag_index, count, self.format_retries(retries))) + self.sleep_retry('fragment', count) def report_skip_fragment(self, frag_index, err=None): err = f' {err};' if err else '' diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 1efdc8957..5c97facb7 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -828,6 +828,18 @@ def create_parser(): '--fragment-retries', dest='fragment_retries', metavar='RETRIES', default=10, help='Number of retries for a fragment (default is %default), or "infinite" (DASH, hlsnative and ISM)') + downloader.add_option( + '--retry-sleep', + dest='retry_sleep', metavar='[TYPE:]EXPR', default={}, type='str', + action='callback', callback=_dict_from_options_callback, + callback_kwargs={ + 'allowed_keys': 'http|fragment|file_access', + 'default_key': 'http', + }, help=( + 'An expression for the time to sleep between retries in seconds (optionally) prefixed ' + 'by the type of retry (http (default), fragment, file_access) to apply the sleep to. ' + 'EXPR can be a number, or of the forms linear=START[:END[:STEP=1]] or exp=START[:END[:BASE=2]]. ' + 'Eg: --retry-sleep linear=1::2 --retry-sleep fragment:exp=1:20')) downloader.add_option( '--skip-unavailable-fragments', '--no-abort-on-unavailable-fragment', action='store_true', dest='skip_unavailable_fragments', default=True,