Add option --throttled-rate below which video data is re-extracted

Currently only for HTTP downloads

Closes #430, workaround for https://github.com/ytdl-org/youtube-dl/issues/29326

yt_dlp/YoutubeDL.py

@@ -101,6 +101,7 @@ from .utils import (
     str_or_none,
     strftime_or_none,
     subtitles_filename,
+    ThrottledDownload,
     to_high_limit_path,
     traverse_obj,
     UnavailableVideoError,
@@ -398,10 +399,9 @@ class YoutubeDL(object):
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the downloader (see yt_dlp/downloader/common.py):
-    nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
+    nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
-    noresizebuffer, retries, continuedl, noprogress, consoletitle,
+    max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle,
-    xattr_set_filesize, external_downloader_args, hls_use_mpegts,
+    xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size.
-    http_chunk_size.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If False, use avconv instead of ffmpeg if both are available,
@@ -1145,6 +1145,10 @@ class YoutubeDL(object):
                 self.report_error(msg)
             except ExtractorError as e:  # An error we somewhat expected
                 self.report_error(compat_str(e), e.format_traceback())
+            except ThrottledDownload:
+                self.to_stderr('\r')
+                self.report_warning('The download speed is below throttle limit. Re-extracting data')
+                return wrapper(self, *args, **kwargs)
             except (MaxDownloadsReached, ExistingVideoReached, RejectedVideoReached):
                 raise
             except Exception as e:

yt_dlp/__init__.py

@@ -151,6 +151,11 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error('invalid rate limit specified')
         opts.ratelimit = numeric_limit
+    if opts.throttledratelimit is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.throttledratelimit)
+        if numeric_limit is None:
+            parser.error('invalid rate limit specified')
+        opts.throttledratelimit = numeric_limit
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
@@ -552,6 +557,7 @@ def _real_main(argv=None):
         'ignoreerrors': opts.ignoreerrors,
         'force_generic_extractor': opts.force_generic_extractor,
         'ratelimit': opts.ratelimit,
+        'throttledratelimit': opts.throttledratelimit,
         'overwrites': opts.overwrites,
         'retries': opts.retries,
         'fragment_retries': opts.fragment_retries,

yt_dlp/downloader/common.py

@@ -14,6 +14,7 @@ from ..utils import (
     format_bytes,
     shell_quote,
     timeconvert,
+    ThrottledDownload,
 )
 
 
@@ -32,6 +33,7 @@ class FileDownloader(object):
     verbose:            Print additional info to stdout.
     quiet:              Do not print messages to stdout.
     ratelimit:          Download speed limit, in bytes/sec.
+    throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
     retries:            Number of times to retry for HTTP error 5xx
     buffersize:         Size of download buffer in bytes.
     noresizebuffer:     Do not automatically resize the download buffer.
@@ -170,7 +172,7 @@ class FileDownloader(object):
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit')
-        if rate_limit is None or byte_counter == 0:
+        if byte_counter == 0:
             return
         if now is None:
             now = time.time()
@@ -178,7 +180,7 @@ class FileDownloader(object):
         if elapsed <= 0.0:
             return
         speed = float(byte_counter) / elapsed
-        if speed > rate_limit:
+        if rate_limit is not None and speed > rate_limit:
             sleep_time = float(byte_counter) / rate_limit - elapsed
             if sleep_time > 0:
                 time.sleep(sleep_time)

yt_dlp/downloader/http.py

@@ -18,6 +18,7 @@ from ..utils import (
     int_or_none,
     sanitize_open,
     sanitized_Request,
+    ThrottledDownload,
     write_xattr,
     XAttrMetadataError,
     XAttrUnavailableError,
@@ -223,6 +224,7 @@ class HttpFD(FileDownloader):
             # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
             now = None  # needed for slow_down() in the first loop run
             before = start  # start measuring
+            throttle_start = None
 
             def retry(e):
                 to_stdout = ctx.tmpfilename == '-'
@@ -313,6 +315,18 @@ class HttpFD(FileDownloader):
                 if data_len is not None and byte_counter == data_len:
                     break
 
+                if speed and speed < (self.params.get('throttledratelimit') or 0):
+                    # The speed must stay below the limit for 3 seconds
+                    # This prevents raising error when the speed temporarily goes down
+                    if throttle_start is None:
+                        throttle_start = now
+                    elif now - throttle_start > 3:
+                        if ctx.stream is not None and ctx.tmpfilename != '-':
+                            ctx.stream.close()
+                        raise ThrottledDownload()
+                else:
+                    throttle_start = None
+
             if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
                 ctx.resume_len = byte_counter
                 # ctx.block_size = block_size

yt_dlp/options.py

@@ -599,6 +599,10 @@ def parseOpts(overrideArguments=None):
         '-r', '--limit-rate', '--rate-limit',
         dest='ratelimit', metavar='RATE',
         help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)')
+    downloader.add_option(
+        '--throttled-rate',
+        dest='throttledratelimit', metavar='RATE',
+        help='Minimum download rate in bytes per second below which throttling is assumed and the video data is re-extracted (e.g. 100K)')
     downloader.add_option(
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,

yt_dlp/utils.py

@@ -2504,6 +2504,11 @@ class RejectedVideoReached(YoutubeDLError):
     pass
 
 
+class ThrottledDownload(YoutubeDLError):
+    """ Download speed below --throttled-rate. """
+    pass
+
+
 class MaxDownloadsReached(YoutubeDLError):
     """ --max-downloads limit has been reached. """
     pass