[yt-dlp.git] / youtube_dl / downloader / http.py

from __future__ import unicode_literals

import errno
import os
import socket
import time
import re

from .common import FileDownloader
from ..compat import (
    compat_urllib_request,
    compat_urllib_error,
)
from ..utils import (
    ContentTooShortError,
    encodeFilename,
    sanitize_open,
)


class HttpFD(FileDownloader):
    def real_download(self, filename, info_dict):
        url = info_dict['url']
        tmpfilename = self.temp_name(filename)
        stream = None

        # Do not include the Accept-Encoding header
        headers = {'Youtubedl-no-compression': 'True'}
        add_headers = info_dict.get('http_headers')
        if add_headers:
            headers.update(add_headers)
        basic_request = compat_urllib_request.Request(url, None, headers)
        request = compat_urllib_request.Request(url, None, headers)

        is_test = self.params.get('test', False)

        if is_test:
            request.add_header('Range', 'bytes=0-%s' % str(self._TEST_FILE_SIZE - 1))

        # Establish possible resume length
        if os.path.isfile(encodeFilename(tmpfilename)):
            resume_len = os.path.getsize(encodeFilename(tmpfilename))
        else:
            resume_len = 0

        open_mode = 'wb'
        if resume_len != 0:
            if self.params.get('continuedl', True):
                self.report_resuming_byte(resume_len)
                request.add_header('Range', 'bytes=%d-' % resume_len)
                open_mode = 'ab'
            else:
                resume_len = 0

        count = 0
        retries = self.params.get('retries', 0)
        while count <= retries:
            # Establish connection
            try:
                data = self.ydl.urlopen(request)
                if resume_len > 0:
                    content_range = data.headers.get('Content-Range')
                    if content_range:
                        content_range_m = re.search(r'bytes (\d+)-', content_range)
                        # Content-Range is correct - go on
                    if content_range_m and resume_len == int(content_range_m.group(1)):
                        break
                    # Content-Range is invalid - wipe the file and do entire redownload
                    self.report_unable_to_resume()
                    resume_len = 0
                    open_mode = 'wb'
                break
            except (compat_urllib_error.HTTPError, ) as err:
                if (err.code < 500 or err.code >= 600) and err.code != 416:
                    # Unexpected HTTP error
                    raise
                elif err.code == 416:
                    # Unable to resume (requested range not satisfiable)
                    try:
                        # Open the connection again without the range header
                        data = self.ydl.urlopen(basic_request)
                        content_length = data.info()['Content-Length']
                    except (compat_urllib_error.HTTPError, ) as err:
                        if err.code < 500 or err.code >= 600:
                            raise
                    else:
                        # Examine the reported length
                        if (content_length is not None and
                                (resume_len - 100 < int(content_length) < resume_len + 100)):
                            # The file had already been fully downloaded.
                            # Explanation to the above condition: in issue #175 it was revealed that
                            # YouTube sometimes adds or removes a few bytes from the end of the file,
                            # changing the file size slightly and causing problems for some users. So
                            # I decided to implement a suggested change and consider the file
                            # completely downloaded if the file size differs less than 100 bytes from
                            # the one in the hard drive.
                            self.report_file_already_downloaded(filename)
                            self.try_rename(tmpfilename, filename)
                            self._hook_progress({
                                'filename': filename,
                                'status': 'finished',
                                'downloaded_bytes': resume_len,
                                'total_bytes': resume_len,
                            })
                            return True
                        else:
                            # The length does not match, we start the download over
                            self.report_unable_to_resume()
                            resume_len = 0
                            open_mode = 'wb'
                            break
            except socket.error as e:
                if e.errno != errno.ECONNRESET:
                    # Connection reset is no problem, just retry
                    raise

            # Retry
            count += 1
            if count <= retries:
                self.report_retry(count, retries)

        if count > retries:
            self.report_error('giving up after %s retries' % retries)
            return False

        data_len = data.info().get('Content-length', None)

        # Range HTTP header may be ignored/unsupported by a webserver
        # (e.g. extractor/scivee.py, extractor/bambuser.py).
        # However, for a test we still would like to download just a piece of a file.
        # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
        # block size when downloading a file.
        if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
            data_len = self._TEST_FILE_SIZE

        if data_len is not None:
            data_len = int(data_len) + resume_len
            min_data_len = self.params.get("min_filesize", None)
            max_data_len = self.params.get("max_filesize", None)
            if min_data_len is not None and data_len < min_data_len:
                self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
                return False
            if max_data_len is not None and data_len > max_data_len:
                self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
                return False

        byte_counter = 0 + resume_len
        block_size = self.params.get('buffersize', 1024)
        start = time.time()

        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
        now = None  # needed for slow_down() in the first loop run
        before = start  # start measuring
        while True:

            # Download and write
            data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
            byte_counter += len(data_block)

            # exit loop when download is finished
            if len(data_block) == 0:
                break

            # Open destination file just in time
            if stream is None:
                try:
                    (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
                    assert stream is not None
                    filename = self.undo_temp_name(tmpfilename)
                    self.report_destination(filename)
                except (OSError, IOError) as err:
                    self.report_error('unable to open for writing: %s' % str(err))
                    return False

                if self.params.get('xattr_set_filesize', False) and data_len is not None:
                    try:
                        import xattr
                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
                    except(OSError, IOError, ImportError) as err:
                        self.report_error('unable to set filesize xattr: %s' % str(err))

            try:
                stream.write(data_block)
            except (IOError, OSError) as err:
                self.to_stderr('\n')
                self.report_error('unable to write data: %s' % str(err))
                return False

            # Apply rate limit
            self.slow_down(start, now, byte_counter - resume_len)

            # end measuring of one loop run
            now = time.time()
            after = now

            # Adjust block size
            if not self.params.get('noresizebuffer', False):
                block_size = self.best_block_size(after - before, len(data_block))

            before = after

            # Progress message
            speed = self.calc_speed(start, now, byte_counter - resume_len)
            if data_len is None:
                eta = None
            else:
                eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)

            self._hook_progress({
                'status': 'downloading',
                'downloaded_bytes': byte_counter,
                'total_bytes': data_len,
                'tmpfilename': tmpfilename,
                'filename': filename,
                'eta': eta,
                'speed': speed,
                'elapsed': now - start,
            })

            if is_test and byte_counter == data_len:
                break

        if stream is None:
            self.to_stderr('\n')
            self.report_error('Did not get any data blocks')
            return False
        if tmpfilename != '-':
            stream.close()

        if data_len is not None and byte_counter != data_len:
            raise ContentTooShortError(byte_counter, int(data_len))
        self.try_rename(tmpfilename, filename)

        # Update file modification time
        if self.params.get('updatetime', True):
            info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))

        self._hook_progress({
            'downloaded_bytes': byte_counter,
            'total_bytes': byte_counter,
            'filename': filename,
            'status': 'finished',
            'elapsed': time.time() - start,
        })

        return True
Commit	Line	Data
4d2d638d PH	1	from __future__ import unicode_literals
4d2d638d PH	2
16e7711e	3	import errno
3bc2ddcc	4	import os
16e7711e	5	import socket
3bc2ddcc	6	import time
8d5b8b47	7	import re
3bc2ddcc JMF	8
3bc2ddcc JMF	9	from .common import FileDownloader
1cc79574	10	from ..compat import (
3bc2ddcc JMF	11	compat_urllib_request,
3bc2ddcc JMF	12	compat_urllib_error,
1cc79574 PH	13	)
1cc79574 PH	14	from ..utils import (
3bc2ddcc	15	ContentTooShortError,
3bc2ddcc JMF	16	encodeFilename,
3bc2ddcc JMF	17	sanitize_open,
3bc2ddcc JMF	18	)
	19
	20
	21	class HttpFD(FileDownloader):
	22	def real_download(self, filename, info_dict):
	23	url = info_dict['url']
	24	tmpfilename = self.temp_name(filename)
	25	stream = None
	26
	27	# Do not include the Accept-Encoding header
	28	headers = {'Youtubedl-no-compression': 'True'}
d769be6c PH	29	add_headers = info_dict.get('http_headers')
	30	if add_headers:
	31	headers.update(add_headers)
2ddcd881 JMF	32	basic_request = compat_urllib_request.Request(url, None, headers)
2ddcd881 JMF	33	request = compat_urllib_request.Request(url, None, headers)
3bc2ddcc	34
df297c87 S	35	is_test = self.params.get('test', False)
	36
	37	if is_test:
	38	request.add_header('Range', 'bytes=0-%s' % str(self._TEST_FILE_SIZE - 1))
3bc2ddcc JMF	39
	40	# Establish possible resume length
	41	if os.path.isfile(encodeFilename(tmpfilename)):
	42	resume_len = os.path.getsize(encodeFilename(tmpfilename))
	43	else:
	44	resume_len = 0
	45
	46	open_mode = 'wb'
	47	if resume_len != 0:
c59e701e	48	if self.params.get('continuedl', True):
3bc2ddcc	49	self.report_resuming_byte(resume_len)
f89197d7	50	request.add_header('Range', 'bytes=%d-' % resume_len)
3bc2ddcc JMF	51	open_mode = 'ab'
	52	else:
	53	resume_len = 0
	54
	55	count = 0
	56	retries = self.params.get('retries', 0)
	57	while count <= retries:
	58	# Establish connection
	59	try:
22ff5d21	60	data = self.ydl.urlopen(request)
d7d2a9a3	61	if resume_len > 0:
	62	content_range = data.headers.get('Content-Range')
	63	if content_range:
	64	content_range_m = re.search(r'bytes (\d+)-', content_range)
c3124c30 S	65	# Content-Range is correct - go on
	66	if content_range_m and resume_len == int(content_range_m.group(1)):
	67	break
d7d2a9a3	68	# Content-Range is invalid - wipe the file and do entire redownload
10eaa8ef	69	self.report_unable_to_resume()
d7d2a9a3	70	resume_len = 0
d7d2a9a3	71	open_mode = 'wb'
3bc2ddcc JMF	72	break
	73	except (compat_urllib_error.HTTPError, ) as err:
	74	if (err.code < 500 or err.code >= 600) and err.code != 416:
	75	# Unexpected HTTP error
	76	raise
	77	elif err.code == 416:
	78	# Unable to resume (requested range not satisfiable)
	79	try:
	80	# Open the connection again without the range header
22ff5d21	81	data = self.ydl.urlopen(basic_request)
3bc2ddcc JMF	82	content_length = data.info()['Content-Length']
	83	except (compat_urllib_error.HTTPError, ) as err:
	84	if err.code < 500 or err.code >= 600:
	85	raise
	86	else:
	87	# Examine the reported length
	88	if (content_length is not None and
	89	(resume_len - 100 < int(content_length) < resume_len + 100)):
	90	# The file had already been fully downloaded.
	91	# Explanation to the above condition: in issue #175 it was revealed that
	92	# YouTube sometimes adds or removes a few bytes from the end of the file,
	93	# changing the file size slightly and causing problems for some users. So
	94	# I decided to implement a suggested change and consider the file
	95	# completely downloaded if the file size differs less than 100 bytes from
	96	# the one in the hard drive.
	97	self.report_file_already_downloaded(filename)
	98	self.try_rename(tmpfilename, filename)
	99	self._hook_progress({
	100	'filename': filename,
	101	'status': 'finished',
f28fe669 JMF	102	'downloaded_bytes': resume_len,
f28fe669 JMF	103	'total_bytes': resume_len,
3bc2ddcc JMF	104	})
	105	return True
	106	else:
	107	# The length does not match, we start the download over
	108	self.report_unable_to_resume()
60daf7f0	109	resume_len = 0
3bc2ddcc JMF	110	open_mode = 'wb'
3bc2ddcc JMF	111	break
16e7711e	112	except socket.error as e:
e77d2975	113	if e.errno != errno.ECONNRESET:
12832049 PH	114	# Connection reset is no problem, just retry
	115	raise
	116
3bc2ddcc JMF	117	# Retry
	118	count += 1
	119	if count <= retries:
	120	self.report_retry(count, retries)
	121
	122	if count > retries:
4d2d638d	123	self.report_error('giving up after %s retries' % retries)
3bc2ddcc JMF	124	return False
	125
	126	data_len = data.info().get('Content-length', None)
df297c87 S	127
	128	# Range HTTP header may be ignored/unsupported by a webserver
	129	# (e.g. extractor/scivee.py, extractor/bambuser.py).
	130	# However, for a test we still would like to download just a piece of a file.
	131	# To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
	132	# block size when downloading a file.
32fd27ec	133	if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
df297c87 S	134	data_len = self._TEST_FILE_SIZE
df297c87 S	135
3bc2ddcc JMF	136	if data_len is not None:
	137	data_len = int(data_len) + resume_len
	138	min_data_len = self.params.get("min_filesize", None)
f89197d7	139	max_data_len = self.params.get("max_filesize", None)
3bc2ddcc	140	if min_data_len is not None and data_len < min_data_len:
4d2d638d	141	self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
3bc2ddcc JMF	142	return False
3bc2ddcc JMF	143	if max_data_len is not None and data_len > max_data_len:
4d2d638d	144	self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
3bc2ddcc JMF	145	return False
3bc2ddcc JMF	146
3bc2ddcc JMF	147	byte_counter = 0 + resume_len
	148	block_size = self.params.get('buffersize', 1024)
	149	start = time.time()
c7667c2d S	150
	151	# measure time over whole while-loop, so slow_down() and best_block_size() work together properly
	152	now = None # needed for slow_down() in the first loop run
	153	before = start # start measuring
3bc2ddcc	154	while True:
c7667c2d	155
3bc2ddcc	156	# Download and write
df297c87	157	data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
c7667c2d S	158	byte_counter += len(data_block)
	159
	160	# exit loop when download is finished
3bc2ddcc JMF	161	if len(data_block) == 0:
3bc2ddcc JMF	162	break
3bc2ddcc	163
c7667c2d	164	# Open destination file just in time
3bc2ddcc JMF	165	if stream is None:
	166	try:
	167	(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
	168	assert stream is not None
	169	filename = self.undo_temp_name(tmpfilename)
	170	self.report_destination(filename)
	171	except (OSError, IOError) as err:
4d2d638d	172	self.report_error('unable to open for writing: %s' % str(err))
3bc2ddcc	173	return False
881e6a1f PH	174
	175	if self.params.get('xattr_set_filesize', False) and data_len is not None:
	176	try:
	177	import xattr
	178	xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
	179	except(OSError, IOError, ImportError) as err:
	180	self.report_error('unable to set filesize xattr: %s' % str(err))
	181
3bc2ddcc JMF	182	try:
3bc2ddcc JMF	183	stream.write(data_block)
7976fcac	184	except (IOError, OSError) as err:
4d2d638d PH	185	self.to_stderr('\n')
4d2d638d PH	186	self.report_error('unable to write data: %s' % str(err))
3bc2ddcc	187	return False
c7667c2d S	188
	189	# Apply rate limit
	190	self.slow_down(start, now, byte_counter - resume_len)
	191
	192	# end measuring of one loop run
	193	now = time.time()
	194	after = now
	195
	196	# Adjust block size
3bc2ddcc JMF	197	if not self.params.get('noresizebuffer', False):
	198	block_size = self.best_block_size(after - before, len(data_block))
	199
c7667c2d S	200	before = after
c7667c2d S	201
3bc2ddcc	202	# Progress message
c7667c2d	203	speed = self.calc_speed(start, now, byte_counter - resume_len)
3bc2ddcc	204	if data_len is None:
a91a2c1a	205	eta = None
3bc2ddcc	206	else:
3bc2ddcc	207	eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
3bc2ddcc JMF	208
3bc2ddcc JMF	209	self._hook_progress({
5cda4eda	210	'status': 'downloading',
3bc2ddcc JMF	211	'downloaded_bytes': byte_counter,
	212	'total_bytes': data_len,
	213	'tmpfilename': tmpfilename,
	214	'filename': filename,
3bc2ddcc JMF	215	'eta': eta,
3bc2ddcc JMF	216	'speed': speed,
5cda4eda	217	'elapsed': now - start,
3bc2ddcc JMF	218	})
3bc2ddcc JMF	219
df297c87 S	220	if is_test and byte_counter == data_len:
	221	break
	222
3bc2ddcc	223	if stream is None:
4d2d638d PH	224	self.to_stderr('\n')
4d2d638d PH	225	self.report_error('Did not get any data blocks')
3bc2ddcc	226	return False
4d2d638d	227	if tmpfilename != '-':
88a23aef	228	stream.close()
5cda4eda	229
3bc2ddcc JMF	230	if data_len is not None and byte_counter != data_len:
	231	raise ContentTooShortError(byte_counter, int(data_len))
	232	self.try_rename(tmpfilename, filename)
	233
	234	# Update file modification time
	235	if self.params.get('updatetime', True):
	236	info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
	237
	238	self._hook_progress({
	239	'downloaded_bytes': byte_counter,
	240	'total_bytes': byte_counter,
	241	'filename': filename,
	242	'status': 'finished',
5cda4eda	243	'elapsed': time.time() - start,
3bc2ddcc JMF	244	})
	245
	246	return True