kumi/yt-dlp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[downloader/http] Retry on more errors (#3065)

Browse source 
Closes #3056, #2071
Related: #3034, #2969
Authored-by: coletdjnz
This commit is contained in:
coletdev 2022-03-19 11:10:20 +13:00 committed by GitHub
parent 510809f1aa
commit a2e77303e3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
yt_dlp/downloader/http.py
View file

@ -1,8 +1,7 @@
from __future__ import unicode_literals
import errno
import os
import socket
import ssl
import time
import random
@ -10,6 +9,7 @@ from .common import FileDownloader
from ..compat import (
compat_str,
compat_urllib_error,
compat_http_client
)
from ..utils import (
ContentTooShortError,
@ -23,6 +23,8 @@ from ..utils import (
XAttrUnavailableError,
)
RESPONSE_READ_EXCEPTIONS = (TimeoutError, ConnectionError, ssl.SSLError, compat_http_client.HTTPException)
class HttpFD(FileDownloader):
def real_download(self, filename, info_dict):
@ -125,14 +127,7 @@ class HttpFD(FileDownloader):
set_range(request, range_start, range_end)
# Establish connection
try:
try:
ctx.data = self.ydl.urlopen(request)
except (compat_urllib_error.URLError, ) as err:
# reason may not be available, e.g. for urllib2.HTTPError on python 2.6
reason = getattr(err, 'reason', None)
if isinstance(reason, socket.timeout):
raise RetryDownload(err)
raise err
ctx.data = self.ydl.urlopen(request)
# When trying to resume, Content-Range HTTP header of response has to be checked
# to match the value of requested Range HTTP header. This is due to a webservers
# that don't support resuming and serve a whole file with no Content-Range
@ -202,13 +197,14 @@ class HttpFD(FileDownloader):
# Unexpected HTTP error
raise
raise RetryDownload(err)
except socket.timeout as err:
except compat_urllib_error.URLError as err:
if isinstance(err.reason, ssl.CertificateError):
raise
raise RetryDownload(err)
# In urllib.request.AbstractHTTPHandler, the response is partially read on request.
# Any errors that occur during this will not be wrapped by URLError
except RESPONSE_READ_EXCEPTIONS as err:
raise RetryDownload(err)
except socket.error as err:
if err.errno in (errno.ECONNRESET, errno.ETIMEDOUT):
# Connection reset is no problem, just retry
raise RetryDownload(err)
raise
def download():
nonlocal throttle_start
@ -254,16 +250,8 @@ class HttpFD(FileDownloader):
try:
# Download and write
data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
# socket.timeout is a subclass of socket.error but may not have
# errno set
except socket.timeout as e:
retry(e)
except socket.error as e:
# SSLError on python 2 (inherits socket.error) may have
# no errno set but this error message
if e.errno in (errno.ECONNRESET, errno.ETIMEDOUT) or getattr(e, 'message', None) == 'The read operation timed out':
retry(e)
raise
except RESPONSE_READ_EXCEPTIONS as err:
retry(err)
byte_counter += len(data_block)