-from __future__ import unicode_literals
-
import os
-import ssl
-import time
import random
+import time
+import urllib.error
from .common import FileDownloader
-from ..compat import (
- compat_str,
- compat_urllib_error,
- compat_http_client
-)
+from ..networking.exceptions import CertificateVerifyError, TransportError
from ..utils import (
ContentTooShortError,
+ RetryManager,
+ ThrottledDownload,
+ XAttrMetadataError,
+ XAttrUnavailableError,
encodeFilename,
int_or_none,
parse_http_range,
sanitized_Request,
- ThrottledDownload,
+ try_call,
write_xattr,
- XAttrMetadataError,
- XAttrUnavailableError,
)
-RESPONSE_READ_EXCEPTIONS = (TimeoutError, ConnectionError, ssl.SSLError, compat_http_client.HTTPException)
-
class HttpFD(FileDownloader):
def real_download(self, filename, info_dict):
ctx.tmpfilename = self.temp_name(filename)
ctx.stream = None
- # Do not include the Accept-Encoding header
- headers = {'Youtubedl-no-compression': 'True'}
+ # Disable compression
+ headers = {'Accept-Encoding': 'identity'}
add_headers = info_dict.get('http_headers')
if add_headers:
headers.update(add_headers)
ctx.open_mode = 'wb'
ctx.resume_len = 0
- ctx.data_len = None
ctx.block_size = self.params.get('buffersize', 1024)
ctx.start_time = time.time()
- ctx.chunk_size = None
- throttle_start = None
# parse given Range
req_start, req_end, _ = parse_http_range(headers.get('Range'))
ctx.is_resume = ctx.resume_len > 0
- count = 0
- retries = self.params.get('retries', 0)
-
class SucceedDownload(Exception):
pass
class NextFragment(Exception):
pass
- def set_range(req, start, end):
- range_header = 'bytes=%d-' % start
- if end:
- range_header += compat_str(end)
- req.add_header('Range', range_header)
-
def establish_connection():
ctx.chunk_size = (random.randint(int(chunk_size * 0.95), chunk_size)
if not is_test and chunk_size else chunk_size)
if ctx.is_resume:
self.report_resuming_byte(ctx.resume_len)
ctx.open_mode = 'ab'
+ elif req_start is not None:
+ range_start = req_start
elif ctx.chunk_size > 0:
range_start = 0
else:
else:
range_end = None
- if range_end and ctx.data_len is not None and range_end >= ctx.data_len:
- range_end = ctx.data_len - 1
- has_range = range_start is not None
- ctx.has_range = has_range
+ if try_call(lambda: range_start > range_end):
+ ctx.resume_len = 0
+ ctx.open_mode = 'wb'
+ raise RetryDownload(Exception(f'Conflicting range. (start={range_start} > end={range_end})'))
+
+ if try_call(lambda: range_end >= ctx.content_len):
+ range_end = ctx.content_len - 1
+
request = sanitized_Request(url, request_data, headers)
+ has_range = range_start is not None
if has_range:
- set_range(request, range_start, range_end)
+ request.add_header('Range', f'bytes={int(range_start)}-{int_or_none(range_end) or ""}')
# Establish connection
try:
ctx.data = self.ydl.urlopen(request)
if has_range:
content_range = ctx.data.headers.get('Content-Range')
content_range_start, content_range_end, content_len = parse_http_range(content_range)
- if content_range_start is not None and range_start == content_range_start:
- # Content-Range is present and matches requested Range, resume is possible
- accept_content_len = (
+ # Content-Range is present and matches requested Range, resume is possible
+ if range_start == content_range_start and (
# Non-chunked download
not ctx.chunk_size
# Chunked download and requested piece or
# its part is promised to be served
or content_range_end == range_end
- or content_len < range_end)
- if accept_content_len:
- ctx.data_len = content_len
- return
+ or content_len < range_end):
+ ctx.content_len = content_len
+ if content_len or req_end:
+ ctx.data_len = min(content_len or req_end, req_end or content_len) - (req_start or 0)
+ return
# Content-Range is either not present or invalid. Assuming remote webserver is
# trying to send the whole file, resume is not possible, so wiping the local file
# and performing entire redownload
- self.report_unable_to_resume()
+ elif range_start > 0:
+ self.report_unable_to_resume()
ctx.resume_len = 0
ctx.open_mode = 'wb'
- ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))
- return
- except (compat_urllib_error.HTTPError, ) as err:
+ ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
+ except urllib.error.HTTPError as err:
if err.code == 416:
# Unable to resume (requested range not satisfiable)
try:
ctx.data = self.ydl.urlopen(
sanitized_Request(url, request_data, headers))
content_length = ctx.data.info()['Content-Length']
- except (compat_urllib_error.HTTPError, ) as err:
+ except urllib.error.HTTPError as err:
if err.code < 500 or err.code >= 600:
raise
else:
# Unexpected HTTP error
raise
raise RetryDownload(err)
- except compat_urllib_error.URLError as err:
- if isinstance(err.reason, ssl.CertificateError):
- raise
- raise RetryDownload(err)
- # In urllib.request.AbstractHTTPHandler, the response is partially read on request.
- # Any errors that occur during this will not be wrapped by URLError
- except RESPONSE_READ_EXCEPTIONS as err:
+ except CertificateVerifyError:
+ raise
+ except TransportError as err:
raise RetryDownload(err)
+ def close_stream():
+ if ctx.stream is not None:
+ if not ctx.tmpfilename == '-':
+ ctx.stream.close()
+ ctx.stream = None
+
def download():
- nonlocal throttle_start
- data_len = ctx.data.info().get('Content-length', None)
+ data_len = ctx.data.info().get('Content-length')
+
+ if ctx.data.info().get('Content-encoding'):
+ # Content-encoding is present, Content-length is not reliable anymore as we are
+ # doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
+ data_len = None
# Range HTTP header may be ignored/unsupported by a webserver
# (e.g. extractor/scivee.py, extractor/bambuser.py).
min_data_len = self.params.get('min_filesize')
max_data_len = self.params.get('max_filesize')
if min_data_len is not None and data_len < min_data_len:
- self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+ self.to_screen(
+ f'\r[download] File is smaller than min-filesize ({data_len} bytes < {min_data_len} bytes). Aborting.')
return False
if max_data_len is not None and data_len > max_data_len:
- self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+ self.to_screen(
+ f'\r[download] File is larger than max-filesize ({data_len} bytes > {max_data_len} bytes). Aborting.')
return False
byte_counter = 0 + ctx.resume_len
before = start # start measuring
def retry(e):
- to_stdout = ctx.tmpfilename == '-'
- if ctx.stream is not None:
- if not to_stdout:
- ctx.stream.close()
- ctx.stream = None
- ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
+ close_stream()
+ ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
+ else os.path.getsize(encodeFilename(ctx.tmpfilename)))
raise RetryDownload(e)
while True:
try:
# Download and write
data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
- except RESPONSE_READ_EXCEPTIONS as err:
+ except TransportError as err:
retry(err)
byte_counter += len(data_block)
assert ctx.stream is not None
ctx.filename = self.undo_temp_name(ctx.tmpfilename)
self.report_destination(ctx.filename)
- except (OSError, IOError) as err:
+ except OSError as err:
self.report_error('unable to open for writing: %s' % str(err))
return False
if self.params.get('xattr_set_filesize', False) and data_len is not None:
try:
- write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode('utf-8'))
+ write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode())
except (XAttrUnavailableError, XAttrMetadataError) as err:
self.report_error('unable to set filesize xattr: %s' % str(err))
try:
ctx.stream.write(data_block)
- except (IOError, OSError) as err:
+ except OSError as err:
self.to_stderr('\n')
self.report_error('unable to write data: %s' % str(err))
return False
if speed and speed < (self.params.get('throttledratelimit') or 0):
# The speed must stay below the limit for 3 seconds
# This prevents raising error when the speed temporarily goes down
- if throttle_start is None:
- throttle_start = now
- elif now - throttle_start > 3:
+ if ctx.throttle_start is None:
+ ctx.throttle_start = now
+ elif now - ctx.throttle_start > 3:
if ctx.stream is not None and ctx.tmpfilename != '-':
ctx.stream.close()
raise ThrottledDownload()
elif speed:
- throttle_start = None
-
- if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len:
- ctx.resume_len = byte_counter
- # ctx.block_size = block_size
- raise NextFragment()
+ ctx.throttle_start = None
if ctx.stream is None:
self.to_stderr('\n')
self.report_error('Did not get any data blocks')
return False
+
+ if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
+ ctx.resume_len = byte_counter
+ raise NextFragment()
+
if ctx.tmpfilename != '-':
ctx.stream.close()
if data_len is not None and byte_counter != data_len:
err = ContentTooShortError(byte_counter, int(data_len))
- if count <= retries:
- retry(err)
- raise err
+ retry(err)
self.try_rename(ctx.tmpfilename, ctx.filename)
return True
- while count <= retries:
+ for retry in RetryManager(self.params.get('retries'), self.report_retry):
try:
establish_connection()
return download()
- except RetryDownload as e:
- count += 1
- if count <= retries:
- self.report_retry(e.source_error, count, retries)
- else:
- self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
+ except RetryDownload as err:
+ retry.error = err.source_error
continue
except NextFragment:
+ retry.error = None
+ retry.attempt -= 1
continue
except SucceedDownload:
return True
-
- self.report_error('giving up after %s retries' % retries)
+ except: # noqa: E722
+ close_stream()
+ raise
return False
projects / yt-dlp.git / blobdiff