[yt-dlp.git] / yt_dlp / downloader / http.py

import http.client
import os
import random
import socket
import ssl
import time
import urllib.error

from .common import FileDownloader
from ..utils import (
    ContentTooShortError,
    ThrottledDownload,
    XAttrMetadataError,
    XAttrUnavailableError,
    encodeFilename,
    int_or_none,
    parse_http_range,
    sanitized_Request,
    try_call,
    write_xattr,
)

RESPONSE_READ_EXCEPTIONS = (
    TimeoutError,
    socket.timeout,  # compat: py < 3.10
    ConnectionError,
    ssl.SSLError,
    http.client.HTTPException
)


class HttpFD(FileDownloader):
    def real_download(self, filename, info_dict):
        url = info_dict['url']
        request_data = info_dict.get('request_data', None)

        class DownloadContext(dict):
            __getattr__ = dict.get
            __setattr__ = dict.__setitem__
            __delattr__ = dict.__delitem__

        ctx = DownloadContext()
        ctx.filename = filename
        ctx.tmpfilename = self.temp_name(filename)
        ctx.stream = None

        # Do not include the Accept-Encoding header
        headers = {'Youtubedl-no-compression': 'True'}
        add_headers = info_dict.get('http_headers')
        if add_headers:
            headers.update(add_headers)

        is_test = self.params.get('test', False)
        chunk_size = self._TEST_FILE_SIZE if is_test else (
            self.params.get('http_chunk_size')
            or info_dict.get('downloader_options', {}).get('http_chunk_size')
            or 0)

        ctx.open_mode = 'wb'
        ctx.resume_len = 0
        ctx.block_size = self.params.get('buffersize', 1024)
        ctx.start_time = time.time()

        # parse given Range
        req_start, req_end, _ = parse_http_range(headers.get('Range'))

        if self.params.get('continuedl', True):
            # Establish possible resume length
            if os.path.isfile(encodeFilename(ctx.tmpfilename)):
                ctx.resume_len = os.path.getsize(
                    encodeFilename(ctx.tmpfilename))

        ctx.is_resume = ctx.resume_len > 0

        count = 0
        retries = self.params.get('retries', 0)

        class SucceedDownload(Exception):
            pass

        class RetryDownload(Exception):
            def __init__(self, source_error):
                self.source_error = source_error

        class NextFragment(Exception):
            pass

        def establish_connection():
            ctx.chunk_size = (random.randint(int(chunk_size * 0.95), chunk_size)
                              if not is_test and chunk_size else chunk_size)
            if ctx.resume_len > 0:
                range_start = ctx.resume_len
                if req_start is not None:
                    # offset the beginning of Range to be within request
                    range_start += req_start
                if ctx.is_resume:
                    self.report_resuming_byte(ctx.resume_len)
                ctx.open_mode = 'ab'
            elif req_start is not None:
                range_start = req_start
            elif ctx.chunk_size > 0:
                range_start = 0
            else:
                range_start = None
            ctx.is_resume = False

            if ctx.chunk_size:
                chunk_aware_end = range_start + ctx.chunk_size - 1
                # we're not allowed to download outside Range
                range_end = chunk_aware_end if req_end is None else min(chunk_aware_end, req_end)
            elif req_end is not None:
                # there's no need for chunked downloads, so download until the end of Range
                range_end = req_end
            else:
                range_end = None

            if try_call(lambda: range_start > range_end):
                ctx.resume_len = 0
                ctx.open_mode = 'wb'
                raise RetryDownload(Exception(f'Conflicting range. (start={range_start} > end={range_end})'))

            if try_call(lambda: range_end >= ctx.content_len):
                range_end = ctx.content_len - 1

            request = sanitized_Request(url, request_data, headers)
            has_range = range_start is not None
            if has_range:
                request.add_header('Range', f'bytes={int(range_start)}-{int_or_none(range_end) or ""}')
            # Establish connection
            try:
                ctx.data = self.ydl.urlopen(request)
                # When trying to resume, Content-Range HTTP header of response has to be checked
                # to match the value of requested Range HTTP header. This is due to a webservers
                # that don't support resuming and serve a whole file with no Content-Range
                # set in response despite of requested Range (see
                # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
                if has_range:
                    content_range = ctx.data.headers.get('Content-Range')
                    content_range_start, content_range_end, content_len = parse_http_range(content_range)
                    # Content-Range is present and matches requested Range, resume is possible
                    if range_start == content_range_start and (
                            # Non-chunked download
                            not ctx.chunk_size
                            # Chunked download and requested piece or
                            # its part is promised to be served
                            or content_range_end == range_end
                            or content_len < range_end):
                        ctx.content_len = content_len
                        if content_len or req_end:
                            ctx.data_len = min(content_len or req_end, req_end or content_len) - (req_start or 0)
                        return
                    # Content-Range is either not present or invalid. Assuming remote webserver is
                    # trying to send the whole file, resume is not possible, so wiping the local file
                    # and performing entire redownload
                    self.report_unable_to_resume()
                    ctx.resume_len = 0
                    ctx.open_mode = 'wb'
                ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
            except urllib.error.HTTPError as err:
                if err.code == 416:
                    # Unable to resume (requested range not satisfiable)
                    try:
                        # Open the connection again without the range header
                        ctx.data = self.ydl.urlopen(
                            sanitized_Request(url, request_data, headers))
                        content_length = ctx.data.info()['Content-Length']
                    except urllib.error.HTTPError as err:
                        if err.code < 500 or err.code >= 600:
                            raise
                    else:
                        # Examine the reported length
                        if (content_length is not None
                                and (ctx.resume_len - 100 < int(content_length) < ctx.resume_len + 100)):
                            # The file had already been fully downloaded.
                            # Explanation to the above condition: in issue #175 it was revealed that
                            # YouTube sometimes adds or removes a few bytes from the end of the file,
                            # changing the file size slightly and causing problems for some users. So
                            # I decided to implement a suggested change and consider the file
                            # completely downloaded if the file size differs less than 100 bytes from
                            # the one in the hard drive.
                            self.report_file_already_downloaded(ctx.filename)
                            self.try_rename(ctx.tmpfilename, ctx.filename)
                            self._hook_progress({
                                'filename': ctx.filename,
                                'status': 'finished',
                                'downloaded_bytes': ctx.resume_len,
                                'total_bytes': ctx.resume_len,
                            }, info_dict)
                            raise SucceedDownload()
                        else:
                            # The length does not match, we start the download over
                            self.report_unable_to_resume()
                            ctx.resume_len = 0
                            ctx.open_mode = 'wb'
                            return
                elif err.code < 500 or err.code >= 600:
                    # Unexpected HTTP error
                    raise
                raise RetryDownload(err)
            except urllib.error.URLError as err:
                if isinstance(err.reason, ssl.CertificateError):
                    raise
                raise RetryDownload(err)
            # In urllib.request.AbstractHTTPHandler, the response is partially read on request.
            # Any errors that occur during this will not be wrapped by URLError
            except RESPONSE_READ_EXCEPTIONS as err:
                raise RetryDownload(err)

        def close_stream():
            if ctx.stream is not None:
                if not ctx.tmpfilename == '-':
                    ctx.stream.close()
                ctx.stream = None

        def download():
            data_len = ctx.data.info().get('Content-length', None)

            # Range HTTP header may be ignored/unsupported by a webserver
            # (e.g. extractor/scivee.py, extractor/bambuser.py).
            # However, for a test we still would like to download just a piece of a file.
            # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
            # block size when downloading a file.
            if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
                data_len = self._TEST_FILE_SIZE

            if data_len is not None:
                data_len = int(data_len) + ctx.resume_len
                min_data_len = self.params.get('min_filesize')
                max_data_len = self.params.get('max_filesize')
                if min_data_len is not None and data_len < min_data_len:
                    self.to_screen(
                        f'\r[download] File is smaller than min-filesize ({data_len} bytes < {min_data_len} bytes). Aborting.')
                    return False
                if max_data_len is not None and data_len > max_data_len:
                    self.to_screen(
                        f'\r[download] File is larger than max-filesize ({data_len} bytes > {max_data_len} bytes). Aborting.')
                    return False

            byte_counter = 0 + ctx.resume_len
            block_size = ctx.block_size
            start = time.time()

            # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
            now = None  # needed for slow_down() in the first loop run
            before = start  # start measuring

            def retry(e):
                close_stream()
                ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
                                  else os.path.getsize(encodeFilename(ctx.tmpfilename)))
                raise RetryDownload(e)

            while True:
                try:
                    # Download and write
                    data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
                except RESPONSE_READ_EXCEPTIONS as err:
                    retry(err)

                byte_counter += len(data_block)

                # exit loop when download is finished
                if len(data_block) == 0:
                    break

                # Open destination file just in time
                if ctx.stream is None:
                    try:
                        ctx.stream, ctx.tmpfilename = self.sanitize_open(
                            ctx.tmpfilename, ctx.open_mode)
                        assert ctx.stream is not None
                        ctx.filename = self.undo_temp_name(ctx.tmpfilename)
                        self.report_destination(ctx.filename)
                    except OSError as err:
                        self.report_error('unable to open for writing: %s' % str(err))
                        return False

                    if self.params.get('xattr_set_filesize', False) and data_len is not None:
                        try:
                            write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode())
                        except (XAttrUnavailableError, XAttrMetadataError) as err:
                            self.report_error('unable to set filesize xattr: %s' % str(err))

                try:
                    ctx.stream.write(data_block)
                except OSError as err:
                    self.to_stderr('\n')
                    self.report_error('unable to write data: %s' % str(err))
                    return False

                # Apply rate limit
                self.slow_down(start, now, byte_counter - ctx.resume_len)

                # end measuring of one loop run
                now = time.time()
                after = now

                # Adjust block size
                if not self.params.get('noresizebuffer', False):
                    block_size = self.best_block_size(after - before, len(data_block))

                before = after

                # Progress message
                speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
                if ctx.data_len is None:
                    eta = None
                else:
                    eta = self.calc_eta(start, time.time(), ctx.data_len - ctx.resume_len, byte_counter - ctx.resume_len)

                self._hook_progress({
                    'status': 'downloading',
                    'downloaded_bytes': byte_counter,
                    'total_bytes': ctx.data_len,
                    'tmpfilename': ctx.tmpfilename,
                    'filename': ctx.filename,
                    'eta': eta,
                    'speed': speed,
                    'elapsed': now - ctx.start_time,
                    'ctx_id': info_dict.get('ctx_id'),
                }, info_dict)

                if data_len is not None and byte_counter == data_len:
                    break

                if speed and speed < (self.params.get('throttledratelimit') or 0):
                    # The speed must stay below the limit for 3 seconds
                    # This prevents raising error when the speed temporarily goes down
                    if ctx.throttle_start is None:
                        ctx.throttle_start = now
                    elif now - ctx.throttle_start > 3:
                        if ctx.stream is not None and ctx.tmpfilename != '-':
                            ctx.stream.close()
                        raise ThrottledDownload()
                elif speed:
                    ctx.throttle_start = None

            if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
                ctx.resume_len = byte_counter
                # ctx.block_size = block_size
                raise NextFragment()

            if ctx.stream is None:
                self.to_stderr('\n')
                self.report_error('Did not get any data blocks')
                return False
            if ctx.tmpfilename != '-':
                ctx.stream.close()

            if data_len is not None and byte_counter != data_len:
                err = ContentTooShortError(byte_counter, int(data_len))
                if count <= retries:
                    retry(err)
                raise err

            self.try_rename(ctx.tmpfilename, ctx.filename)

            # Update file modification time
            if self.params.get('updatetime', True):
                info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))

            self._hook_progress({
                'downloaded_bytes': byte_counter,
                'total_bytes': byte_counter,
                'filename': ctx.filename,
                'status': 'finished',
                'elapsed': time.time() - ctx.start_time,
                'ctx_id': info_dict.get('ctx_id'),
            }, info_dict)

            return True

        while count <= retries:
            try:
                establish_connection()
                return download()
            except RetryDownload as e:
                count += 1
                if count <= retries:
                    self.report_retry(e.source_error, count, retries)
                else:
                    self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
                continue
            except NextFragment:
                continue
            except SucceedDownload:
                return True
            except:  # noqa: E722
                close_stream()
                raise

        self.report_error('giving up after %s retries' % retries)
        return False
Commit	Line	Data
54007a45	1	import http.client
3bc2ddcc	2	import os
f8271158	3	import random
5faf6528	4	import socket
a2e77303	5	import ssl
3bc2ddcc	6	import time
ac668111	7	import urllib.error
3bc2ddcc JMF	8
3bc2ddcc JMF	9	from .common import FileDownloader
1cc79574	10	from ..utils import (
3bc2ddcc	11	ContentTooShortError,
f8271158	12	ThrottledDownload,
	13	XAttrMetadataError,
	14	XAttrUnavailableError,
3bc2ddcc	15	encodeFilename,
ba515388	16	int_or_none,
93c8410d	17	parse_http_range,
5c2266df	18	sanitized_Request,
c4f60dd7	19	try_call,
d7753d19	20	write_xattr,
3bc2ddcc JMF	21	)
3bc2ddcc JMF	22
5faf6528	23	RESPONSE_READ_EXCEPTIONS = (
	24	TimeoutError,
	25	socket.timeout, # compat: py < 3.10
	26	ConnectionError,
	27	ssl.SSLError,
ac668111	28	http.client.HTTPException
5faf6528	29	)
a2e77303	30
3bc2ddcc JMF	31
3bc2ddcc JMF	32	class HttpFD(FileDownloader):
d3f0687c	33	def real_download(self, filename, info_dict):
3bc2ddcc	34	url = info_dict['url']
273762c8	35	request_data = info_dict.get('request_data', None)
a3c3a1e1 S	36
	37	class DownloadContext(dict):
	38	__getattr__ = dict.get
	39	__setattr__ = dict.__setitem__
	40	__delattr__ = dict.__delitem__
	41
	42	ctx = DownloadContext()
	43	ctx.filename = filename
	44	ctx.tmpfilename = self.temp_name(filename)
	45	ctx.stream = None
3bc2ddcc JMF	46
	47	# Do not include the Accept-Encoding header
	48	headers = {'Youtubedl-no-compression': 'True'}
d769be6c PH	49	add_headers = info_dict.get('http_headers')
	50	if add_headers:
	51	headers.update(add_headers)
3bc2ddcc	52
df297c87	53	is_test = self.params.get('test', False)
ba515388	54	chunk_size = self._TEST_FILE_SIZE if is_test else (
b5ae35ee	55	self.params.get('http_chunk_size')
b5ae35ee	56	or info_dict.get('downloader_options', {}).get('http_chunk_size')
b922db9f	57	or 0)
3bc2ddcc	58
a3c3a1e1 S	59	ctx.open_mode = 'wb'
a3c3a1e1 S	60	ctx.resume_len = 0
ba515388 S	61	ctx.block_size = self.params.get('buffersize', 1024)
ba515388 S	62	ctx.start_time = time.time()
a3c3a1e1	63
93c8410d LNO	64	# parse given Range
	65	req_start, req_end, _ = parse_http_range(headers.get('Range'))
	66
a3c3a1e1 S	67	if self.params.get('continuedl', True):
	68	# Establish possible resume length
	69	if os.path.isfile(encodeFilename(ctx.tmpfilename)):
ba515388 S	70	ctx.resume_len = os.path.getsize(
	71	encodeFilename(ctx.tmpfilename))
	72
	73	ctx.is_resume = ctx.resume_len > 0
3bc2ddcc JMF	74
	75	count = 0
	76	retries = self.params.get('retries', 0)
a3c3a1e1 S	77
	78	class SucceedDownload(Exception):
	79	pass
	80
	81	class RetryDownload(Exception):
	82	def __init__(self, source_error):
	83	self.source_error = source_error
	84
ba515388 S	85	class NextFragment(Exception):
	86	pass
	87
a3c3a1e1	88	def establish_connection():
b91a7a4e S	89	ctx.chunk_size = (random.randint(int(chunk_size * 0.95), chunk_size)
b91a7a4e S	90	if not is_test and chunk_size else chunk_size)
ba515388 S	91	if ctx.resume_len > 0:
ba515388 S	92	range_start = ctx.resume_len
93c8410d LNO	93	if req_start is not None:
	94	# offset the beginning of Range to be within request
	95	range_start += req_start
ba515388 S	96	if ctx.is_resume:
ba515388 S	97	self.report_resuming_byte(ctx.resume_len)
a3c3a1e1	98	ctx.open_mode = 'ab'
be4685ab LNO	99	elif req_start is not None:
be4685ab LNO	100	range_start = req_start
b91a7a4e	101	elif ctx.chunk_size > 0:
ba515388 S	102	range_start = 0
	103	else:
	104	range_start = None
	105	ctx.is_resume = False
93c8410d LNO	106
	107	if ctx.chunk_size:
	108	chunk_aware_end = range_start + ctx.chunk_size - 1
	109	# we're not allowed to download outside Range
	110	range_end = chunk_aware_end if req_end is None else min(chunk_aware_end, req_end)
	111	elif req_end is not None:
	112	# there's no need for chunked downloads, so download until the end of Range
	113	range_end = req_end
	114	else:
	115	range_end = None
	116
c4f60dd7	117	if try_call(lambda: range_start > range_end):
be4685ab LNO	118	ctx.resume_len = 0
	119	ctx.open_mode = 'wb'
	120	raise RetryDownload(Exception(f'Conflicting range. (start={range_start} > end={range_end})'))
	121
c4f60dd7	122	if try_call(lambda: range_end >= ctx.content_len):
be4685ab LNO	123	range_end = ctx.content_len - 1
be4685ab LNO	124
273762c8	125	request = sanitized_Request(url, request_data, headers)
be4685ab	126	has_range = range_start is not None
ba515388	127	if has_range:
a44ca5a4	128	request.add_header('Range', f'bytes={int(range_start)}-{int_or_none(range_end) or ""}')
3bc2ddcc JMF	129	# Establish connection
3bc2ddcc JMF	130	try:
a2e77303	131	ctx.data = self.ydl.urlopen(request)
84bc4dcb S	132	# When trying to resume, Content-Range HTTP header of response has to be checked
	133	# to match the value of requested Range HTTP header. This is due to a webservers
	134	# that don't support resuming and serve a whole file with no Content-Range
	135	# set in response despite of requested Range (see
067aa17e	136	# https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
ba515388	137	if has_range:
a3c3a1e1	138	content_range = ctx.data.headers.get('Content-Range')
93c8410d	139	content_range_start, content_range_end, content_len = parse_http_range(content_range)
c487cf00	140	# Content-Range is present and matches requested Range, resume is possible
c487cf00	141	if range_start == content_range_start and (
93c8410d LNO	142	# Non-chunked download
	143	not ctx.chunk_size
	144	# Chunked download and requested piece or
	145	# its part is promised to be served
	146	or content_range_end == range_end
c487cf00	147	or content_len < range_end):
	148	ctx.content_len = content_len
	149	if content_len or req_end:
	150	ctx.data_len = min(content_len or req_end, req_end or content_len) - (req_start or 0)
	151	return
84bc4dcb S	152	# Content-Range is either not present or invalid. Assuming remote webserver is
	153	# trying to send the whole file, resume is not possible, so wiping the local file
	154	# and performing entire redownload
10eaa8ef	155	self.report_unable_to_resume()
a3c3a1e1 S	156	ctx.resume_len = 0
a3c3a1e1 S	157	ctx.open_mode = 'wb'
be4685ab	158	ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
ac668111	159	except urllib.error.HTTPError as err:
ba515388	160	if err.code == 416:
3bc2ddcc JMF	161	# Unable to resume (requested range not satisfiable)
	162	try:
	163	# Open the connection again without the range header
cf7259bc	164	ctx.data = self.ydl.urlopen(
273762c8	165	sanitized_Request(url, request_data, headers))
a3c3a1e1	166	content_length = ctx.data.info()['Content-Length']
ac668111	167	except urllib.error.HTTPError as err:
3bc2ddcc JMF	168	if err.code < 500 or err.code >= 600:
	169	raise
	170	else:
	171	# Examine the reported length
3089bc74 S	172	if (content_length is not None
3089bc74 S	173	and (ctx.resume_len - 100 < int(content_length) < ctx.resume_len + 100)):
3bc2ddcc JMF	174	# The file had already been fully downloaded.
	175	# Explanation to the above condition: in issue #175 it was revealed that
	176	# YouTube sometimes adds or removes a few bytes from the end of the file,
	177	# changing the file size slightly and causing problems for some users. So
	178	# I decided to implement a suggested change and consider the file
	179	# completely downloaded if the file size differs less than 100 bytes from
	180	# the one in the hard drive.
a3c3a1e1 S	181	self.report_file_already_downloaded(ctx.filename)
a3c3a1e1 S	182	self.try_rename(ctx.tmpfilename, ctx.filename)
3bc2ddcc	183	self._hook_progress({
a3c3a1e1	184	'filename': ctx.filename,
3bc2ddcc	185	'status': 'finished',
a3c3a1e1 S	186	'downloaded_bytes': ctx.resume_len,
a3c3a1e1 S	187	'total_bytes': ctx.resume_len,
3ba7740d	188	}, info_dict)
a3c3a1e1	189	raise SucceedDownload()
3bc2ddcc JMF	190	else:
	191	# The length does not match, we start the download over
	192	self.report_unable_to_resume()
a3c3a1e1 S	193	ctx.resume_len = 0
	194	ctx.open_mode = 'wb'
	195	return
ba515388 S	196	elif err.code < 500 or err.code >= 600:
	197	# Unexpected HTTP error
	198	raise
a3c3a1e1	199	raise RetryDownload(err)
ac668111	200	except urllib.error.URLError as err:
a2e77303	201	if isinstance(err.reason, ssl.CertificateError):
	202	raise
	203	raise RetryDownload(err)
	204	# In urllib.request.AbstractHTTPHandler, the response is partially read on request.
	205	# Any errors that occur during this will not be wrapped by URLError
	206	except RESPONSE_READ_EXCEPTIONS as err:
9fab498f	207	raise RetryDownload(err)
a3c3a1e1	208
6d645b55	209	def close_stream():
	210	if ctx.stream is not None:
	211	if not ctx.tmpfilename == '-':
	212	ctx.stream.close()
	213	ctx.stream = None
	214
a3c3a1e1 S	215	def download():
	216	data_len = ctx.data.info().get('Content-length', None)
	217
	218	# Range HTTP header may be ignored/unsupported by a webserver
	219	# (e.g. extractor/scivee.py, extractor/bambuser.py).
	220	# However, for a test we still would like to download just a piece of a file.
	221	# To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
	222	# block size when downloading a file.
	223	if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
	224	data_len = self._TEST_FILE_SIZE
	225
	226	if data_len is not None:
	227	data_len = int(data_len) + ctx.resume_len
	228	min_data_len = self.params.get('min_filesize')
	229	max_data_len = self.params.get('max_filesize')
	230	if min_data_len is not None and data_len < min_data_len:
86e5f3ed	231	self.to_screen(
86e5f3ed	232	f'\r[download] File is smaller than min-filesize ({data_len} bytes < {min_data_len} bytes). Aborting.')
a3c3a1e1 S	233	return False
a3c3a1e1 S	234	if max_data_len is not None and data_len > max_data_len:
86e5f3ed	235	self.to_screen(
86e5f3ed	236	f'\r[download] File is larger than max-filesize ({data_len} bytes > {max_data_len} bytes). Aborting.')
a3c3a1e1	237	return False
12832049	238
a3c3a1e1	239	byte_counter = 0 + ctx.resume_len
ba515388	240	block_size = ctx.block_size
a3c3a1e1	241	start = time.time()
c7667c2d	242
a3c3a1e1 S	243	# measure time over whole while-loop, so slow_down() and best_block_size() work together properly
	244	now = None # needed for slow_down() in the first loop run
	245	before = start # start measuring
c7667c2d	246
a3c3a1e1	247	def retry(e):
6d645b55	248	close_stream()
	249	ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
	250	else os.path.getsize(encodeFilename(ctx.tmpfilename)))
a3c3a1e1	251	raise RetryDownload(e)
c7667c2d	252
a3c3a1e1 S	253	while True:
	254	try:
	255	# Download and write
0715f7e1	256	data_block = ctx.data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
a2e77303	257	except RESPONSE_READ_EXCEPTIONS as err:
a2e77303	258	retry(err)
a3c3a1e1 S	259
	260	byte_counter += len(data_block)
	261
	262	# exit loop when download is finished
	263	if len(data_block) == 0:
	264	break
	265
	266	# Open destination file just in time
	267	if ctx.stream is None:
	268	try:
205a0654	269	ctx.stream, ctx.tmpfilename = self.sanitize_open(
a3c3a1e1 S	270	ctx.tmpfilename, ctx.open_mode)
	271	assert ctx.stream is not None
	272	ctx.filename = self.undo_temp_name(ctx.tmpfilename)
	273	self.report_destination(ctx.filename)
86e5f3ed	274	except OSError as err:
a3c3a1e1 S	275	self.report_error('unable to open for writing: %s' % str(err))
	276	return False
	277
	278	if self.params.get('xattr_set_filesize', False) and data_len is not None:
	279	try:
0f06bcd7	280	write_xattr(ctx.tmpfilename, 'user.ytdl.filesize', str(data_len).encode())
a3c3a1e1 S	281	except (XAttrUnavailableError, XAttrMetadataError) as err:
a3c3a1e1 S	282	self.report_error('unable to set filesize xattr: %s' % str(err))
3bc2ddcc	283
3bc2ddcc	284	try:
a3c3a1e1	285	ctx.stream.write(data_block)
86e5f3ed	286	except OSError as err:
a3c3a1e1 S	287	self.to_stderr('\n')
a3c3a1e1 S	288	self.report_error('unable to write data: %s' % str(err))
3bc2ddcc	289	return False
881e6a1f	290
a3c3a1e1 S	291	# Apply rate limit
	292	self.slow_down(start, now, byte_counter - ctx.resume_len)
	293
	294	# end measuring of one loop run
	295	now = time.time()
	296	after = now
	297
	298	# Adjust block size
	299	if not self.params.get('noresizebuffer', False):
	300	block_size = self.best_block_size(after - before, len(data_block))
	301
	302	before = after
	303
	304	# Progress message
	305	speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
ba515388	306	if ctx.data_len is None:
a3c3a1e1 S	307	eta = None
a3c3a1e1 S	308	else:
ba515388	309	eta = self.calc_eta(start, time.time(), ctx.data_len - ctx.resume_len, byte_counter - ctx.resume_len)
a3c3a1e1 S	310
	311	self._hook_progress({
	312	'status': 'downloading',
	313	'downloaded_bytes': byte_counter,
ba515388	314	'total_bytes': ctx.data_len,
a3c3a1e1 S	315	'tmpfilename': ctx.tmpfilename,
	316	'filename': ctx.filename,
	317	'eta': eta,
	318	'speed': speed,
ba515388	319	'elapsed': now - ctx.start_time,
bd50a52b	320	'ctx_id': info_dict.get('ctx_id'),
3ba7740d	321	}, info_dict)
a3c3a1e1	322
f7b42518	323	if data_len is not None and byte_counter == data_len:
a3c3a1e1 S	324	break
a3c3a1e1 S	325
51d9739f	326	if speed and speed < (self.params.get('throttledratelimit') or 0):
	327	# The speed must stay below the limit for 3 seconds
	328	# This prevents raising error when the speed temporarily goes down
a44ca5a4	329	if ctx.throttle_start is None:
	330	ctx.throttle_start = now
	331	elif now - ctx.throttle_start > 3:
51d9739f	332	if ctx.stream is not None and ctx.tmpfilename != '-':
	333	ctx.stream.close()
	334	raise ThrottledDownload()
21186af7	335	elif speed:
a44ca5a4	336	ctx.throttle_start = None
51d9739f	337
be4685ab	338	if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
ba515388 S	339	ctx.resume_len = byte_counter
	340	# ctx.block_size = block_size
	341	raise NextFragment()
	342
a3c3a1e1	343	if ctx.stream is None:
4d2d638d	344	self.to_stderr('\n')
a3c3a1e1	345	self.report_error('Did not get any data blocks')
3bc2ddcc	346	return False
a3c3a1e1 S	347	if ctx.tmpfilename != '-':
a3c3a1e1 S	348	ctx.stream.close()
c7667c2d	349
a3c3a1e1 S	350	if data_len is not None and byte_counter != data_len:
	351	err = ContentTooShortError(byte_counter, int(data_len))
	352	if count <= retries:
	353	retry(err)
	354	raise err
c7667c2d	355
a3c3a1e1	356	self.try_rename(ctx.tmpfilename, ctx.filename)
c7667c2d	357
a3c3a1e1 S	358	# Update file modification time
	359	if self.params.get('updatetime', True):
	360	info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
3bc2ddcc JMF	361
	362	self._hook_progress({
	363	'downloaded_bytes': byte_counter,
a3c3a1e1 S	364	'total_bytes': byte_counter,
	365	'filename': ctx.filename,
	366	'status': 'finished',
ba515388	367	'elapsed': time.time() - ctx.start_time,
bd50a52b	368	'ctx_id': info_dict.get('ctx_id'),
3ba7740d	369	}, info_dict)
3bc2ddcc	370
a3c3a1e1 S	371	return True
	372
	373	while count <= retries:
	374	try:
	375	establish_connection()
bec49996	376	return download()
a3c3a1e1 S	377	except RetryDownload as e:
	378	count += 1
	379	if count <= retries:
	380	self.report_retry(e.source_error, count, retries)
d5a39f0b	381	else:
d5a39f0b	382	self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
a3c3a1e1	383	continue
ba515388 S	384	except NextFragment:
ba515388 S	385	continue
a3c3a1e1 S	386	except SucceedDownload:
a3c3a1e1 S	387	return True
6d645b55	388	except: # noqa: E722
	389	close_stream()
	390	raise
a3c3a1e1 S	391
	392	self.report_error('giving up after %s retries' % retries)
	393	return False