1 from __future__
import unicode_literals
9 from .common
import FileDownloader
10 from ..compat
import (
11 compat_urllib_request
,
21 class HttpFD(FileDownloader
):
22 def real_download(self
, filename
, info_dict
):
23 url
= info_dict
['url']
24 tmpfilename
= self
.temp_name(filename
)
27 # Do not include the Accept-Encoding header
28 headers
= {'Youtubedl-no-compression': 'True'}
29 add_headers
= info_dict
.get('http_headers')
31 headers
.update(add_headers
)
32 basic_request
= compat_urllib_request
.Request(url
, None, headers
)
33 request
= compat_urllib_request
.Request(url
, None, headers
)
35 is_test
= self
.params
.get('test', False)
38 request
.add_header('Range', 'bytes=0-%s' % str(self
._TEST
_FILE
_SIZE
- 1))
40 # Establish possible resume length
41 if os
.path
.isfile(encodeFilename(tmpfilename
)):
42 resume_len
= os
.path
.getsize(encodeFilename(tmpfilename
))
48 if self
.params
.get('continuedl', True):
49 self
.report_resuming_byte(resume_len
)
50 request
.add_header('Range', 'bytes=%d-' % resume_len
)
56 retries
= self
.params
.get('retries', 0)
57 while count
<= retries
:
58 # Establish connection
60 data
= self
.ydl
.urlopen(request
)
63 content_range
= data
.headers
.get('Content-Range')
65 content_range_m
= re
.search(r
'bytes (\d+)-', content_range
)
67 # Content-Range is correct - go on
68 if resume_len
== int(content_range_m
.group(1)):
71 # Content-Range is invalid - wipe the file and do entire redownload
76 except (compat_urllib_error
.HTTPError
, ) as err
:
77 if (err
.code
< 500 or err
.code
>= 600) and err
.code
!= 416:
78 # Unexpected HTTP error
81 # Unable to resume (requested range not satisfiable)
83 # Open the connection again without the range header
84 data
= self
.ydl
.urlopen(basic_request
)
85 content_length
= data
.info()['Content-Length']
86 except (compat_urllib_error
.HTTPError
, ) as err
:
87 if err
.code
< 500 or err
.code
>= 600:
90 # Examine the reported length
91 if (content_length
is not None and
92 (resume_len
- 100 < int(content_length
) < resume_len
+ 100)):
93 # The file had already been fully downloaded.
94 # Explanation to the above condition: in issue #175 it was revealed that
95 # YouTube sometimes adds or removes a few bytes from the end of the file,
96 # changing the file size slightly and causing problems for some users. So
97 # I decided to implement a suggested change and consider the file
98 # completely downloaded if the file size differs less than 100 bytes from
99 # the one in the hard drive.
100 self
.report_file_already_downloaded(filename
)
101 self
.try_rename(tmpfilename
, filename
)
102 self
._hook
_progress
({
103 'filename': filename
,
104 'status': 'finished',
105 'downloaded_bytes': resume_len
,
106 'total_bytes': resume_len
,
110 # The length does not match, we start the download over
111 self
.report_unable_to_resume()
115 except socket
.error
as e
:
116 if e
.errno
!= errno
.ECONNRESET
:
117 # Connection reset is no problem, just retry
123 self
.report_retry(count
, retries
)
126 self
.report_error('giving up after %s retries' % retries
)
129 data_len
= data
.info().get('Content-length', None)
131 # Range HTTP header may be ignored/unsupported by a webserver
132 # (e.g. extractor/scivee.py, extractor/bambuser.py).
133 # However, for a test we still would like to download just a piece of a file.
134 # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
135 # block size when downloading a file.
136 if is_test
and (data_len
is None or int(data_len
) > self
._TEST
_FILE
_SIZE
):
137 data_len
= self
._TEST
_FILE
_SIZE
139 if data_len
is not None:
140 data_len
= int(data_len
) + resume_len
141 min_data_len
= self
.params
.get("min_filesize", None)
142 max_data_len
= self
.params
.get("max_filesize", None)
143 if min_data_len
is not None and data_len
< min_data_len
:
144 self
.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len
, min_data_len
))
146 if max_data_len
is not None and data_len
> max_data_len
:
147 self
.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len
, max_data_len
))
150 byte_counter
= 0 + resume_len
151 block_size
= self
.params
.get('buffersize', 1024)
154 # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
155 now
= None # needed for slow_down() in the first loop run
156 before
= start
# start measuring
160 data_block
= data
.read(block_size
if not is_test
else min(block_size
, data_len
- byte_counter
))
161 byte_counter
+= len(data_block
)
163 # exit loop when download is finished
164 if len(data_block
) == 0:
167 # Open destination file just in time
170 (stream
, tmpfilename
) = sanitize_open(tmpfilename
, open_mode
)
171 assert stream
is not None
172 filename
= self
.undo_temp_name(tmpfilename
)
173 self
.report_destination(filename
)
174 except (OSError, IOError) as err
:
175 self
.report_error('unable to open for writing: %s' % str(err
))
178 if self
.params
.get('xattr_set_filesize', False) and data_len
is not None:
181 xattr
.setxattr(tmpfilename
, 'user.ytdl.filesize', str(data_len
))
182 except(OSError, IOError, ImportError) as err
:
183 self
.report_error('unable to set filesize xattr: %s' % str(err
))
186 stream
.write(data_block
)
187 except (IOError, OSError) as err
:
189 self
.report_error('unable to write data: %s' % str(err
))
193 self
.slow_down(start
, now
, byte_counter
- resume_len
)
195 # end measuring of one loop run
200 if not self
.params
.get('noresizebuffer', False):
201 block_size
= self
.best_block_size(after
- before
, len(data_block
))
206 speed
= self
.calc_speed(start
, now
, byte_counter
- resume_len
)
210 eta
= self
.calc_eta(start
, time
.time(), data_len
- resume_len
, byte_counter
- resume_len
)
212 self
._hook
_progress
({
213 'status': 'downloading',
214 'downloaded_bytes': byte_counter
,
215 'total_bytes': data_len
,
216 'tmpfilename': tmpfilename
,
217 'filename': filename
,
220 'elapsed': now
- start
,
223 if is_test
and byte_counter
== data_len
:
228 self
.report_error('Did not get any data blocks')
230 if tmpfilename
!= '-':
233 if data_len
is not None and byte_counter
!= data_len
:
234 raise ContentTooShortError(byte_counter
, int(data_len
))
235 self
.try_rename(tmpfilename
, filename
)
237 # Update file modification time
238 if self
.params
.get('updatetime', True):
239 info_dict
['filetime'] = self
.try_utime(filename
, data
.info().get('last-modified', None))
241 self
._hook
_progress
({
242 'downloaded_bytes': byte_counter
,
243 'total_bytes': byte_counter
,
244 'filename': filename
,
245 'status': 'finished',
246 'elapsed': time
.time() - start
,