+import http.client
import os
import random
import socket
import ssl
import time
+import urllib.error
from .common import FileDownloader
-from ..compat import compat_http_client, compat_urllib_error
from ..utils import (
ContentTooShortError,
+ RetryManager,
ThrottledDownload,
XAttrMetadataError,
XAttrUnavailableError,
socket.timeout, # compat: py < 3.10
ConnectionError,
ssl.SSLError,
- compat_http_client.HTTPException
+ http.client.HTTPException
)
ctx.tmpfilename = self.temp_name(filename)
ctx.stream = None
- # Do not include the Accept-Encoding header
- headers = {'Youtubedl-no-compression': 'True'}
+ # Disable compression
+ headers = {'Accept-Encoding': 'identity'}
add_headers = info_dict.get('http_headers')
if add_headers:
headers.update(add_headers)
ctx.is_resume = ctx.resume_len > 0
- count = 0
- retries = self.params.get('retries', 0)
-
class SucceedDownload(Exception):
pass
# Content-Range is either not present or invalid. Assuming remote webserver is
# trying to send the whole file, resume is not possible, so wiping the local file
# and performing entire redownload
- self.report_unable_to_resume()
+ elif range_start > 0:
+ self.report_unable_to_resume()
ctx.resume_len = 0
ctx.open_mode = 'wb'
ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
- except compat_urllib_error.HTTPError as err:
+ except urllib.error.HTTPError as err:
if err.code == 416:
# Unable to resume (requested range not satisfiable)
try:
ctx.data = self.ydl.urlopen(
sanitized_Request(url, request_data, headers))
content_length = ctx.data.info()['Content-Length']
- except compat_urllib_error.HTTPError as err:
+ except urllib.error.HTTPError as err:
if err.code < 500 or err.code >= 600:
raise
else:
# Unexpected HTTP error
raise
raise RetryDownload(err)
- except compat_urllib_error.URLError as err:
+ except urllib.error.URLError as err:
if isinstance(err.reason, ssl.CertificateError):
raise
raise RetryDownload(err)
except RESPONSE_READ_EXCEPTIONS as err:
raise RetryDownload(err)
+ def close_stream():
+ if ctx.stream is not None:
+ if not ctx.tmpfilename == '-':
+ ctx.stream.close()
+ ctx.stream = None
+
def download():
- data_len = ctx.data.info().get('Content-length', None)
+ data_len = ctx.data.info().get('Content-length')
+
+ if ctx.data.info().get('Content-encoding'):
+ # Content-encoding is present, Content-length is not reliable anymore as we are
+ # doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
+ data_len = None
# Range HTTP header may be ignored/unsupported by a webserver
# (e.g. extractor/scivee.py, extractor/bambuser.py).
before = start # start measuring
def retry(e):
- to_stdout = ctx.tmpfilename == '-'
- if ctx.stream is not None:
- if not to_stdout:
- ctx.stream.close()
- ctx.stream = None
- ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
+ close_stream()
+ ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
+ else os.path.getsize(encodeFilename(ctx.tmpfilename)))
raise RetryDownload(e)
while True:
elif speed:
ctx.throttle_start = None
- if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
- ctx.resume_len = byte_counter
- # ctx.block_size = block_size
- raise NextFragment()
-
if ctx.stream is None:
self.to_stderr('\n')
self.report_error('Did not get any data blocks')
return False
+
+ if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
+ ctx.resume_len = byte_counter
+ raise NextFragment()
+
if ctx.tmpfilename != '-':
ctx.stream.close()
if data_len is not None and byte_counter != data_len:
err = ContentTooShortError(byte_counter, int(data_len))
- if count <= retries:
- retry(err)
- raise err
+ retry(err)
self.try_rename(ctx.tmpfilename, ctx.filename)
return True
- while count <= retries:
+ for retry in RetryManager(self.params.get('retries'), self.report_retry):
try:
establish_connection()
return download()
- except RetryDownload as e:
- count += 1
- if count <= retries:
- self.report_retry(e.source_error, count, retries)
- else:
- self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
+ except RetryDownload as err:
+ retry.error = err.source_error
continue
except NextFragment:
+ retry.error = None
+ retry.attempt -= 1
continue
except SucceedDownload:
return True
-
- self.report_error('giving up after %s retries' % retries)
+ except: # noqa: E722
+ close_stream()
+ raise
return False