227bf1a33b
New networking interface consists of a `RequestDirector` that directs each `Request` to appropriate `RequestHandler` and returns the `Response` or raises `RequestError`. The handlers define adapters to transform its internal Request/Response/Errors to our interfaces. User-facing changes: - Fix issues with per request proxies on redirects for urllib - Support for `ALL_PROXY` environment variable for proxy setting - Support for `socks5h` proxy - Closes https://github.com/yt-dlp/yt-dlp/issues/6325, https://github.com/ytdl-org/youtube-dl/issues/22618, https://github.com/ytdl-org/youtube-dl/pull/28093 - Raise error when using `https` proxy instead of silently converting it to `http` Authored by: coletdjnz
458 lines
17 KiB
Python
458 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import functools
|
|
import gzip
|
|
import http.client
|
|
import io
|
|
import socket
|
|
import ssl
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import urllib.response
|
|
import zlib
|
|
from urllib.request import (
|
|
DataHandler,
|
|
FileHandler,
|
|
FTPHandler,
|
|
HTTPCookieProcessor,
|
|
HTTPDefaultErrorHandler,
|
|
HTTPErrorProcessor,
|
|
UnknownHandler,
|
|
)
|
|
|
|
from ._helper import (
|
|
InstanceStoreMixin,
|
|
add_accept_encoding_header,
|
|
get_redirect_method,
|
|
make_socks_proxy_opts,
|
|
select_proxy,
|
|
)
|
|
from .common import Features, RequestHandler, Response, register
|
|
from .exceptions import (
|
|
CertificateVerifyError,
|
|
HTTPError,
|
|
IncompleteRead,
|
|
ProxyError,
|
|
RequestError,
|
|
SSLError,
|
|
TransportError,
|
|
)
|
|
from ..dependencies import brotli
|
|
from ..socks import ProxyError as SocksProxyError
|
|
from ..socks import sockssocket
|
|
from ..utils import escape_url, update_url_query
|
|
|
|
SUPPORTED_ENCODINGS = ['gzip', 'deflate']
|
|
CONTENT_DECODE_ERRORS = [zlib.error, OSError]
|
|
|
|
if brotli:
|
|
SUPPORTED_ENCODINGS.append('br')
|
|
CONTENT_DECODE_ERRORS.append(brotli.error)
|
|
|
|
|
|
def _create_http_connection(http_class, source_address, *args, **kwargs):
|
|
hc = http_class(*args, **kwargs)
|
|
|
|
if source_address is not None:
|
|
# This is to workaround _create_connection() from socket where it will try all
|
|
# address data from getaddrinfo() including IPv6. This filters the result from
|
|
# getaddrinfo() based on the source_address value.
|
|
# This is based on the cpython socket.create_connection() function.
|
|
# https://github.com/python/cpython/blob/master/Lib/socket.py#L691
|
|
def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
|
|
host, port = address
|
|
err = None
|
|
addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
|
|
af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
|
|
ip_addrs = [addr for addr in addrs if addr[0] == af]
|
|
if addrs and not ip_addrs:
|
|
ip_version = 'v4' if af == socket.AF_INET else 'v6'
|
|
raise OSError(
|
|
"No remote IP%s addresses available for connect, can't use '%s' as source address"
|
|
% (ip_version, source_address[0]))
|
|
for res in ip_addrs:
|
|
af, socktype, proto, canonname, sa = res
|
|
sock = None
|
|
try:
|
|
sock = socket.socket(af, socktype, proto)
|
|
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
|
|
sock.settimeout(timeout)
|
|
sock.bind(source_address)
|
|
sock.connect(sa)
|
|
err = None # Explicitly break reference cycle
|
|
return sock
|
|
except OSError as _:
|
|
err = _
|
|
if sock is not None:
|
|
sock.close()
|
|
if err is not None:
|
|
raise err
|
|
else:
|
|
raise OSError('getaddrinfo returns an empty list')
|
|
if hasattr(hc, '_create_connection'):
|
|
hc._create_connection = _create_connection
|
|
hc.source_address = (source_address, 0)
|
|
|
|
return hc
|
|
|
|
|
|
class HTTPHandler(urllib.request.AbstractHTTPHandler):
|
|
"""Handler for HTTP requests and responses.
|
|
|
|
This class, when installed with an OpenerDirector, automatically adds
|
|
the standard headers to every HTTP request and handles gzipped, deflated and
|
|
brotli responses from web servers.
|
|
|
|
Part of this code was copied from:
|
|
|
|
http://techknack.net/python-urllib2-handlers/
|
|
|
|
Andrew Rowls, the author of that code, agreed to release it to the
|
|
public domain.
|
|
"""
|
|
|
|
def __init__(self, context=None, source_address=None, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self._source_address = source_address
|
|
self._context = context
|
|
|
|
@staticmethod
|
|
def _make_conn_class(base, req):
|
|
conn_class = base
|
|
socks_proxy = req.headers.pop('Ytdl-socks-proxy', None)
|
|
if socks_proxy:
|
|
conn_class = make_socks_conn_class(conn_class, socks_proxy)
|
|
return conn_class
|
|
|
|
def http_open(self, req):
|
|
conn_class = self._make_conn_class(http.client.HTTPConnection, req)
|
|
return self.do_open(functools.partial(
|
|
_create_http_connection, conn_class, self._source_address), req)
|
|
|
|
def https_open(self, req):
|
|
conn_class = self._make_conn_class(http.client.HTTPSConnection, req)
|
|
return self.do_open(
|
|
functools.partial(
|
|
_create_http_connection, conn_class, self._source_address),
|
|
req, context=self._context)
|
|
|
|
@staticmethod
|
|
def deflate(data):
|
|
if not data:
|
|
return data
|
|
try:
|
|
return zlib.decompress(data, -zlib.MAX_WBITS)
|
|
except zlib.error:
|
|
return zlib.decompress(data)
|
|
|
|
@staticmethod
|
|
def brotli(data):
|
|
if not data:
|
|
return data
|
|
return brotli.decompress(data)
|
|
|
|
@staticmethod
|
|
def gz(data):
|
|
gz = gzip.GzipFile(fileobj=io.BytesIO(data), mode='rb')
|
|
try:
|
|
return gz.read()
|
|
except OSError as original_oserror:
|
|
# There may be junk add the end of the file
|
|
# See http://stackoverflow.com/q/4928560/35070 for details
|
|
for i in range(1, 1024):
|
|
try:
|
|
gz = gzip.GzipFile(fileobj=io.BytesIO(data[:-i]), mode='rb')
|
|
return gz.read()
|
|
except OSError:
|
|
continue
|
|
else:
|
|
raise original_oserror
|
|
|
|
def http_request(self, req):
|
|
# According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
|
|
# always respected by websites, some tend to give out URLs with non percent-encoded
|
|
# non-ASCII characters (see telemb.py, ard.py [#3412])
|
|
# urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
|
|
# To work around aforementioned issue we will replace request's original URL with
|
|
# percent-encoded one
|
|
# Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
|
|
# the code of this workaround has been moved here from YoutubeDL.urlopen()
|
|
url = req.get_full_url()
|
|
url_escaped = escape_url(url)
|
|
|
|
# Substitute URL if any change after escaping
|
|
if url != url_escaped:
|
|
req = update_Request(req, url=url_escaped)
|
|
|
|
return super().do_request_(req)
|
|
|
|
def http_response(self, req, resp):
|
|
old_resp = resp
|
|
|
|
# Content-Encoding header lists the encodings in order that they were applied [1].
|
|
# To decompress, we simply do the reverse.
|
|
# [1]: https://datatracker.ietf.org/doc/html/rfc9110#name-content-encoding
|
|
decoded_response = None
|
|
for encoding in (e.strip() for e in reversed(resp.headers.get('Content-encoding', '').split(','))):
|
|
if encoding == 'gzip':
|
|
decoded_response = self.gz(decoded_response or resp.read())
|
|
elif encoding == 'deflate':
|
|
decoded_response = self.deflate(decoded_response or resp.read())
|
|
elif encoding == 'br' and brotli:
|
|
decoded_response = self.brotli(decoded_response or resp.read())
|
|
|
|
if decoded_response is not None:
|
|
resp = urllib.request.addinfourl(io.BytesIO(decoded_response), old_resp.headers, old_resp.url, old_resp.code)
|
|
resp.msg = old_resp.msg
|
|
# Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
|
|
# https://github.com/ytdl-org/youtube-dl/issues/6457).
|
|
if 300 <= resp.code < 400:
|
|
location = resp.headers.get('Location')
|
|
if location:
|
|
# As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
|
|
location = location.encode('iso-8859-1').decode()
|
|
location_escaped = escape_url(location)
|
|
if location != location_escaped:
|
|
del resp.headers['Location']
|
|
resp.headers['Location'] = location_escaped
|
|
return resp
|
|
|
|
https_request = http_request
|
|
https_response = http_response
|
|
|
|
|
|
def make_socks_conn_class(base_class, socks_proxy):
|
|
assert issubclass(base_class, (
|
|
http.client.HTTPConnection, http.client.HTTPSConnection))
|
|
|
|
proxy_args = make_socks_proxy_opts(socks_proxy)
|
|
|
|
class SocksConnection(base_class):
|
|
def connect(self):
|
|
self.sock = sockssocket()
|
|
self.sock.setproxy(**proxy_args)
|
|
if type(self.timeout) in (int, float): # noqa: E721
|
|
self.sock.settimeout(self.timeout)
|
|
self.sock.connect((self.host, self.port))
|
|
|
|
if isinstance(self, http.client.HTTPSConnection):
|
|
self.sock = self._context.wrap_socket(self.sock, server_hostname=self.host)
|
|
|
|
return SocksConnection
|
|
|
|
|
|
class RedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
"""YoutubeDL redirect handler
|
|
|
|
The code is based on HTTPRedirectHandler implementation from CPython [1].
|
|
|
|
This redirect handler fixes and improves the logic to better align with RFC7261
|
|
and what browsers tend to do [2][3]
|
|
|
|
1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
|
|
2. https://datatracker.ietf.org/doc/html/rfc7231
|
|
3. https://github.com/python/cpython/issues/91306
|
|
"""
|
|
|
|
http_error_301 = http_error_303 = http_error_307 = http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
if code not in (301, 302, 303, 307, 308):
|
|
raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp)
|
|
|
|
new_data = req.data
|
|
|
|
# Technically the Cookie header should be in unredirected_hdrs,
|
|
# however in practice some may set it in normal headers anyway.
|
|
# We will remove it here to prevent any leaks.
|
|
remove_headers = ['Cookie']
|
|
|
|
new_method = get_redirect_method(req.get_method(), code)
|
|
# only remove payload if method changed (e.g. POST to GET)
|
|
if new_method != req.get_method():
|
|
new_data = None
|
|
remove_headers.extend(['Content-Length', 'Content-Type'])
|
|
|
|
new_headers = {k: v for k, v in req.headers.items() if k.title() not in remove_headers}
|
|
|
|
return urllib.request.Request(
|
|
newurl, headers=new_headers, origin_req_host=req.origin_req_host,
|
|
unverifiable=True, method=new_method, data=new_data)
|
|
|
|
|
|
class ProxyHandler(urllib.request.BaseHandler):
|
|
handler_order = 100
|
|
|
|
def __init__(self, proxies=None):
|
|
self.proxies = proxies
|
|
# Set default handlers
|
|
for type in ('http', 'https', 'ftp'):
|
|
setattr(self, '%s_open' % type, lambda r, meth=self.proxy_open: meth(r))
|
|
|
|
def proxy_open(self, req):
|
|
proxy = select_proxy(req.get_full_url(), self.proxies)
|
|
if proxy is None:
|
|
return
|
|
if urllib.parse.urlparse(proxy).scheme.lower() in ('socks4', 'socks4a', 'socks5', 'socks5h'):
|
|
req.add_header('Ytdl-socks-proxy', proxy)
|
|
# yt-dlp's http/https handlers do wrapping the socket with socks
|
|
return None
|
|
return urllib.request.ProxyHandler.proxy_open(
|
|
self, req, proxy, None)
|
|
|
|
|
|
class PUTRequest(urllib.request.Request):
|
|
def get_method(self):
|
|
return 'PUT'
|
|
|
|
|
|
class HEADRequest(urllib.request.Request):
|
|
def get_method(self):
|
|
return 'HEAD'
|
|
|
|
|
|
def update_Request(req, url=None, data=None, headers=None, query=None):
|
|
req_headers = req.headers.copy()
|
|
req_headers.update(headers or {})
|
|
req_data = data or req.data
|
|
req_url = update_url_query(url or req.get_full_url(), query)
|
|
req_get_method = req.get_method()
|
|
if req_get_method == 'HEAD':
|
|
req_type = HEADRequest
|
|
elif req_get_method == 'PUT':
|
|
req_type = PUTRequest
|
|
else:
|
|
req_type = urllib.request.Request
|
|
new_req = req_type(
|
|
req_url, data=req_data, headers=req_headers,
|
|
origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
|
|
if hasattr(req, 'timeout'):
|
|
new_req.timeout = req.timeout
|
|
return new_req
|
|
|
|
|
|
class UrllibResponseAdapter(Response):
|
|
"""
|
|
HTTP Response adapter class for urllib addinfourl and http.client.HTTPResponse
|
|
"""
|
|
|
|
def __init__(self, res: http.client.HTTPResponse | urllib.response.addinfourl):
|
|
# addinfourl: In Python 3.9+, .status was introduced and .getcode() was deprecated [1]
|
|
# HTTPResponse: .getcode() was deprecated, .status always existed [2]
|
|
# 1. https://docs.python.org/3/library/urllib.request.html#urllib.response.addinfourl.getcode
|
|
# 2. https://docs.python.org/3.10/library/http.client.html#http.client.HTTPResponse.status
|
|
super().__init__(
|
|
fp=res, headers=res.headers, url=res.url,
|
|
status=getattr(res, 'status', None) or res.getcode(), reason=getattr(res, 'reason', None))
|
|
|
|
def read(self, amt=None):
|
|
try:
|
|
return self.fp.read(amt)
|
|
except Exception as e:
|
|
handle_response_read_exceptions(e)
|
|
raise e
|
|
|
|
|
|
def handle_sslerror(e: ssl.SSLError):
|
|
if not isinstance(e, ssl.SSLError):
|
|
return
|
|
if isinstance(e, ssl.SSLCertVerificationError):
|
|
raise CertificateVerifyError(cause=e) from e
|
|
raise SSLError(cause=e) from e
|
|
|
|
|
|
def handle_response_read_exceptions(e):
|
|
if isinstance(e, http.client.IncompleteRead):
|
|
raise IncompleteRead(partial=e.partial, cause=e, expected=e.expected) from e
|
|
elif isinstance(e, ssl.SSLError):
|
|
handle_sslerror(e)
|
|
elif isinstance(e, (OSError, EOFError, http.client.HTTPException, *CONTENT_DECODE_ERRORS)):
|
|
# OSErrors raised here should mostly be network related
|
|
raise TransportError(cause=e) from e
|
|
|
|
|
|
@register
|
|
class UrllibRH(RequestHandler, InstanceStoreMixin):
|
|
_SUPPORTED_URL_SCHEMES = ('http', 'https', 'data', 'ftp')
|
|
_SUPPORTED_PROXY_SCHEMES = ('http', 'socks4', 'socks4a', 'socks5', 'socks5h')
|
|
_SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY)
|
|
RH_NAME = 'urllib'
|
|
|
|
def __init__(self, *, enable_file_urls: bool = False, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.enable_file_urls = enable_file_urls
|
|
if self.enable_file_urls:
|
|
self._SUPPORTED_URL_SCHEMES = (*self._SUPPORTED_URL_SCHEMES, 'file')
|
|
|
|
def _create_instance(self, proxies, cookiejar):
|
|
opener = urllib.request.OpenerDirector()
|
|
handlers = [
|
|
ProxyHandler(proxies),
|
|
HTTPHandler(
|
|
debuglevel=int(bool(self.verbose)),
|
|
context=self._make_sslcontext(),
|
|
source_address=self.source_address),
|
|
HTTPCookieProcessor(cookiejar),
|
|
DataHandler(),
|
|
UnknownHandler(),
|
|
HTTPDefaultErrorHandler(),
|
|
FTPHandler(),
|
|
HTTPErrorProcessor(),
|
|
RedirectHandler(),
|
|
]
|
|
|
|
if self.enable_file_urls:
|
|
handlers.append(FileHandler())
|
|
|
|
for handler in handlers:
|
|
opener.add_handler(handler)
|
|
|
|
# Delete the default user-agent header, which would otherwise apply in
|
|
# cases where our custom HTTP handler doesn't come into play
|
|
# (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
|
|
opener.addheaders = []
|
|
return opener
|
|
|
|
def _send(self, request):
|
|
headers = self._merge_headers(request.headers)
|
|
add_accept_encoding_header(headers, SUPPORTED_ENCODINGS)
|
|
urllib_req = urllib.request.Request(
|
|
url=request.url,
|
|
data=request.data,
|
|
headers=dict(headers),
|
|
method=request.method
|
|
)
|
|
|
|
opener = self._get_instance(
|
|
proxies=request.proxies or self.proxies,
|
|
cookiejar=request.extensions.get('cookiejar') or self.cookiejar
|
|
)
|
|
try:
|
|
res = opener.open(urllib_req, timeout=float(request.extensions.get('timeout') or self.timeout))
|
|
except urllib.error.HTTPError as e:
|
|
if isinstance(e.fp, (http.client.HTTPResponse, urllib.response.addinfourl)):
|
|
# Prevent file object from being closed when urllib.error.HTTPError is destroyed.
|
|
e._closer.file = None
|
|
raise HTTPError(UrllibResponseAdapter(e.fp), redirect_loop='redirect error' in str(e)) from e
|
|
raise # unexpected
|
|
except urllib.error.URLError as e:
|
|
cause = e.reason # NOTE: cause may be a string
|
|
|
|
# proxy errors
|
|
if 'tunnel connection failed' in str(cause).lower() or isinstance(cause, SocksProxyError):
|
|
raise ProxyError(cause=e) from e
|
|
|
|
handle_response_read_exceptions(cause)
|
|
raise TransportError(cause=e) from e
|
|
except (http.client.InvalidURL, ValueError) as e:
|
|
# Validation errors
|
|
# http.client.HTTPConnection raises ValueError in some validation cases
|
|
# such as if request method contains illegal control characters [1]
|
|
# 1. https://github.com/python/cpython/blob/987b712b4aeeece336eed24fcc87a950a756c3e2/Lib/http/client.py#L1256
|
|
raise RequestError(cause=e) from e
|
|
except Exception as e:
|
|
handle_response_read_exceptions(e)
|
|
raise # unexpected
|
|
|
|
return UrllibResponseAdapter(res)
|