1 from __future__
import unicode_literals
9 from .common
import FileDownloader
10 from ..compat
import (
11 compat_urllib_request
,
21 class HttpFD(FileDownloader
):
22 def real_download(self
, filename
, info_dict
):
23 url
= info_dict
['url']
24 tmpfilename
= self
.temp_name(filename
)
27 # Do not include the Accept-Encoding header
28 headers
= {'Youtubedl-no-compression': 'True'}
29 add_headers
= info_dict
.get('http_headers')
31 headers
.update(add_headers
)
32 basic_request
= compat_urllib_request
.Request(url
, None, headers
)
33 request
= compat_urllib_request
.Request(url
, None, headers
)
35 is_test
= self
.params
.get('test', False)
38 request
.add_header('Range', 'bytes=0-%s' % str(self
._TEST
_FILE
_SIZE
- 1))
40 # Establish possible resume length
41 if os
.path
.isfile(encodeFilename(tmpfilename
)):
42 resume_len
= os
.path
.getsize(encodeFilename(tmpfilename
))
48 if self
.params
.get('continuedl', True):
49 self
.report_resuming_byte(resume_len
)
50 request
.add_header('Range', 'bytes=%d-' % resume_len
)
56 retries
= self
.params
.get('retries', 0)
57 while count
<= retries
:
58 # Establish connection
60 data
= self
.ydl
.urlopen(request
)
62 content_range
= data
.headers
.get('Content-Range')
64 content_range_m
= re
.search(r
'bytes (\d+)-', content_range
)
65 # Content-Range is correct - go on
66 if content_range_m
and resume_len
== int(content_range_m
.group(1)):
68 # Content-Range is invalid - wipe the file and do entire redownload
69 self
.report_unable_to_resume()
73 except (compat_urllib_error
.HTTPError
, ) as err
:
74 if (err
.code
< 500 or err
.code
>= 600) and err
.code
!= 416:
75 # Unexpected HTTP error
78 # Unable to resume (requested range not satisfiable)
80 # Open the connection again without the range header
81 data
= self
.ydl
.urlopen(basic_request
)
82 content_length
= data
.info()['Content-Length']
83 except (compat_urllib_error
.HTTPError
, ) as err
:
84 if err
.code
< 500 or err
.code
>= 600:
87 # Examine the reported length
88 if (content_length
is not None and
89 (resume_len
- 100 < int(content_length
) < resume_len
+ 100)):
90 # The file had already been fully downloaded.
91 # Explanation to the above condition: in issue #175 it was revealed that
92 # YouTube sometimes adds or removes a few bytes from the end of the file,
93 # changing the file size slightly and causing problems for some users. So
94 # I decided to implement a suggested change and consider the file
95 # completely downloaded if the file size differs less than 100 bytes from
96 # the one in the hard drive.
97 self
.report_file_already_downloaded(filename
)
98 self
.try_rename(tmpfilename
, filename
)
100 'filename': filename
,
101 'status': 'finished',
102 'downloaded_bytes': resume_len
,
103 'total_bytes': resume_len
,
107 # The length does not match, we start the download over
108 self
.report_unable_to_resume()
112 except socket
.error
as e
:
113 if e
.errno
!= errno
.ECONNRESET
:
114 # Connection reset is no problem, just retry
120 self
.report_retry(count
, retries
)
123 self
.report_error('giving up after %s retries' % retries
)
126 data_len
= data
.info().get('Content-length', None)
128 # Range HTTP header may be ignored/unsupported by a webserver
129 # (e.g. extractor/scivee.py, extractor/bambuser.py).
130 # However, for a test we still would like to download just a piece of a file.
131 # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
132 # block size when downloading a file.
133 if is_test
and (data_len
is None or int(data_len
) > self
._TEST
_FILE
_SIZE
):
134 data_len
= self
._TEST
_FILE
_SIZE
136 if data_len
is not None:
137 data_len
= int(data_len
) + resume_len
138 min_data_len
= self
.params
.get("min_filesize", None)
139 max_data_len
= self
.params
.get("max_filesize", None)
140 if min_data_len
is not None and data_len
< min_data_len
:
141 self
.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len
, min_data_len
))
143 if max_data_len
is not None and data_len
> max_data_len
:
144 self
.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len
, max_data_len
))
147 byte_counter
= 0 + resume_len
148 block_size
= self
.params
.get('buffersize', 1024)
151 # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
152 now
= None # needed for slow_down() in the first loop run
153 before
= start
# start measuring
157 data_block
= data
.read(block_size
if not is_test
else min(block_size
, data_len
- byte_counter
))
158 byte_counter
+= len(data_block
)
160 # exit loop when download is finished
161 if len(data_block
) == 0:
164 # Open destination file just in time
167 (stream
, tmpfilename
) = sanitize_open(tmpfilename
, open_mode
)
168 assert stream
is not None
169 filename
= self
.undo_temp_name(tmpfilename
)
170 self
.report_destination(filename
)
171 except (OSError, IOError) as err
:
172 self
.report_error('unable to open for writing: %s' % str(err
))
175 if self
.params
.get('xattr_set_filesize', False) and data_len
is not None:
178 xattr
.setxattr(tmpfilename
, 'user.ytdl.filesize', str(data_len
))
179 except(OSError, IOError, ImportError) as err
:
180 self
.report_error('unable to set filesize xattr: %s' % str(err
))
183 stream
.write(data_block
)
184 except (IOError, OSError) as err
:
186 self
.report_error('unable to write data: %s' % str(err
))
190 self
.slow_down(start
, now
, byte_counter
- resume_len
)
192 # end measuring of one loop run
197 if not self
.params
.get('noresizebuffer', False):
198 block_size
= self
.best_block_size(after
- before
, len(data_block
))
203 speed
= self
.calc_speed(start
, now
, byte_counter
- resume_len
)
207 eta
= self
.calc_eta(start
, time
.time(), data_len
- resume_len
, byte_counter
- resume_len
)
209 self
._hook
_progress
({
210 'status': 'downloading',
211 'downloaded_bytes': byte_counter
,
212 'total_bytes': data_len
,
213 'tmpfilename': tmpfilename
,
214 'filename': filename
,
217 'elapsed': now
- start
,
220 if is_test
and byte_counter
== data_len
:
225 self
.report_error('Did not get any data blocks')
227 if tmpfilename
!= '-':
230 if data_len
is not None and byte_counter
!= data_len
:
231 raise ContentTooShortError(byte_counter
, int(data_len
))
232 self
.try_rename(tmpfilename
, filename
)
234 # Update file modification time
235 if self
.params
.get('updatetime', True):
236 info_dict
['filetime'] = self
.try_utime(filename
, data
.info().get('last-modified', None))
238 self
._hook
_progress
({
239 'downloaded_bytes': byte_counter
,
240 'total_bytes': byte_counter
,
241 'filename': filename
,
242 'status': 'finished',
243 'elapsed': time
.time() - start
,