-import http.client
import os
import random
-import socket
-import ssl
import time
-import urllib.error
from .common import FileDownloader
+from ..networking import Request
+from ..networking.exceptions import (
+ CertificateVerifyError,
+ HTTPError,
+ TransportError,
+)
from ..utils import (
ContentTooShortError,
RetryManager,
encodeFilename,
int_or_none,
parse_http_range,
- sanitized_Request,
try_call,
write_xattr,
)
-
-RESPONSE_READ_EXCEPTIONS = (
- TimeoutError,
- socket.timeout, # compat: py < 3.10
- ConnectionError,
- ssl.SSLError,
- http.client.HTTPException
-)
+from ..utils.networking import HTTPHeaderDict
class HttpFD(FileDownloader):
ctx.stream = None
# Disable compression
- headers = {'Accept-Encoding': 'identity'}
- add_headers = info_dict.get('http_headers')
- if add_headers:
- headers.update(add_headers)
+ headers = HTTPHeaderDict({'Accept-Encoding': 'identity'}, info_dict.get('http_headers'))
is_test = self.params.get('test', False)
chunk_size = self._TEST_FILE_SIZE if is_test else (
if try_call(lambda: range_end >= ctx.content_len):
range_end = ctx.content_len - 1
- request = sanitized_Request(url, request_data, headers)
+ request = Request(url, request_data, headers)
has_range = range_start is not None
if has_range:
- request.add_header('Range', f'bytes={int(range_start)}-{int_or_none(range_end) or ""}')
+ request.headers['Range'] = f'bytes={int(range_start)}-{int_or_none(range_end) or ""}'
# Establish connection
try:
ctx.data = self.ydl.urlopen(request)
self.report_unable_to_resume()
ctx.resume_len = 0
ctx.open_mode = 'wb'
- ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
- except urllib.error.HTTPError as err:
- if err.code == 416:
+ ctx.data_len = ctx.content_len = int_or_none(ctx.data.headers.get('Content-length', None))
+ except HTTPError as err:
+ if err.status == 416:
# Unable to resume (requested range not satisfiable)
try:
# Open the connection again without the range header
ctx.data = self.ydl.urlopen(
- sanitized_Request(url, request_data, headers))
- content_length = ctx.data.info()['Content-Length']
- except urllib.error.HTTPError as err:
- if err.code < 500 or err.code >= 600:
+ Request(url, request_data, headers))
+ content_length = ctx.data.headers['Content-Length']
+ except HTTPError as err:
+ if err.status < 500 or err.status >= 600:
raise
else:
# Examine the reported length
'downloaded_bytes': ctx.resume_len,
'total_bytes': ctx.resume_len,
}, info_dict)
- raise SucceedDownload()
+ raise SucceedDownload
else:
# The length does not match, we start the download over
self.report_unable_to_resume()
ctx.resume_len = 0
ctx.open_mode = 'wb'
return
- elif err.code < 500 or err.code >= 600:
+ elif err.status < 500 or err.status >= 600:
# Unexpected HTTP error
raise
raise RetryDownload(err)
- except urllib.error.URLError as err:
- if isinstance(err.reason, ssl.CertificateError):
- raise
- raise RetryDownload(err)
- # In urllib.request.AbstractHTTPHandler, the response is partially read on request.
- # Any errors that occur during this will not be wrapped by URLError
- except RESPONSE_READ_EXCEPTIONS as err:
+ except CertificateVerifyError:
+ raise
+ except TransportError as err:
raise RetryDownload(err)
def close_stream():
if ctx.stream is not None:
- if not ctx.tmpfilename == '-':
+ if ctx.tmpfilename != '-':
ctx.stream.close()
ctx.stream = None
def download():
- data_len = ctx.data.info().get('Content-length')
+ data_len = ctx.data.headers.get('Content-length')
- if ctx.data.info().get('Content-encoding'):
+ if ctx.data.headers.get('Content-encoding'):
# Content-encoding is present, Content-length is not reliable anymore as we are
# doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
data_len = None
def retry(e):
close_stream()
- ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
- else os.path.getsize(encodeFilename(ctx.tmpfilename)))
+ if ctx.tmpfilename == '-':
+ ctx.resume_len = byte_counter
+ else:
+ try:
+ ctx.resume_len = os.path.getsize(encodeFilename(ctx.tmpfilename))
+ except FileNotFoundError:
+ ctx.resume_len = 0
raise RetryDownload(e)
while True:
try:
# Download and write
data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
- except RESPONSE_READ_EXCEPTIONS as err:
+ except TransportError as err:
retry(err)
byte_counter += len(data_block)
ctx.filename = self.undo_temp_name(ctx.tmpfilename)
self.report_destination(ctx.filename)
except OSError as err:
- self.report_error('unable to open for writing: %s' % str(err))
+ self.report_error(f'unable to open for writing: {err}')
return False
if self.params.get('xattr_set_filesize', False) and data_len is not None:
try:
write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode())
except (XAttrUnavailableError, XAttrMetadataError) as err:
- self.report_error('unable to set filesize xattr: %s' % str(err))
+ self.report_error(f'unable to set filesize xattr: {err}')
try:
ctx.stream.write(data_block)
except OSError as err:
self.to_stderr('\n')
- self.report_error('unable to write data: %s' % str(err))
+ self.report_error(f'unable to write data: {err}')
return False
# Apply rate limit
elif now - ctx.throttle_start > 3:
if ctx.stream is not None and ctx.tmpfilename != '-':
ctx.stream.close()
- raise ThrottledDownload()
+ raise ThrottledDownload
elif speed:
ctx.throttle_start = None
if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
ctx.resume_len = byte_counter
- raise NextFragment()
+ raise NextFragment
if ctx.tmpfilename != '-':
ctx.stream.close()
# Update file modification time
if self.params.get('updatetime', True):
- info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
+ info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.headers.get('last-modified', None))
self._hook_progress({
'downloaded_bytes': byte_counter,
projects / yt-dlp.git / blobdiff