1 from __future__
import unicode_literals
8 from .common
import FileDownloader
10 compat_urllib_request
,
20 class HttpFD(FileDownloader
):
21 def real_download(self
, filename
, info_dict
):
22 url
= info_dict
['url']
23 tmpfilename
= self
.temp_name(filename
)
26 # Do not include the Accept-Encoding header
27 headers
= {'Youtubedl-no-compression': 'True'}
28 add_headers
= info_dict
.get('http_headers')
30 headers
.update(add_headers
)
31 data
= info_dict
.get('http_post_data')
32 http_method
= info_dict
.get('http_method')
33 basic_request
= compat_urllib_request
.Request(url
, data
, headers
)
34 request
= compat_urllib_request
.Request(url
, data
, headers
)
35 if http_method
is not None:
36 basic_request
.get_method
= lambda: http_method
37 request
.get_method
= lambda: http_method
39 is_test
= self
.params
.get('test', False)
42 request
.add_header('Range', 'bytes=0-%s' % str(self
._TEST
_FILE
_SIZE
- 1))
44 # Establish possible resume length
45 if os
.path
.isfile(encodeFilename(tmpfilename
)):
46 resume_len
= os
.path
.getsize(encodeFilename(tmpfilename
))
52 if self
.params
.get('continuedl', False):
53 self
.report_resuming_byte(resume_len
)
54 request
.add_header('Range', 'bytes=%d-' % resume_len
)
60 retries
= self
.params
.get('retries', 0)
61 while count
<= retries
:
62 # Establish connection
64 data
= self
.ydl
.urlopen(request
)
66 except (compat_urllib_error
.HTTPError
, ) as err
:
67 if (err
.code
< 500 or err
.code
>= 600) and err
.code
!= 416:
68 # Unexpected HTTP error
71 # Unable to resume (requested range not satisfiable)
73 # Open the connection again without the range header
74 data
= self
.ydl
.urlopen(basic_request
)
75 content_length
= data
.info()['Content-Length']
76 except (compat_urllib_error
.HTTPError
, ) as err
:
77 if err
.code
< 500 or err
.code
>= 600:
80 # Examine the reported length
81 if (content_length
is not None and
82 (resume_len
- 100 < int(content_length
) < resume_len
+ 100)):
83 # The file had already been fully downloaded.
84 # Explanation to the above condition: in issue #175 it was revealed that
85 # YouTube sometimes adds or removes a few bytes from the end of the file,
86 # changing the file size slightly and causing problems for some users. So
87 # I decided to implement a suggested change and consider the file
88 # completely downloaded if the file size differs less than 100 bytes from
89 # the one in the hard drive.
90 self
.report_file_already_downloaded(filename
)
91 self
.try_rename(tmpfilename
, filename
)
95 'downloaded_bytes': resume_len
,
96 'total_bytes': resume_len
,
100 # The length does not match, we start the download over
101 self
.report_unable_to_resume()
105 except socket
.error
as e
:
106 if e
.errno
!= errno
.ECONNRESET
:
107 # Connection reset is no problem, just retry
113 self
.report_retry(count
, retries
)
116 self
.report_error('giving up after %s retries' % retries
)
119 data_len
= data
.info().get('Content-length', None)
121 # Range HTTP header may be ignored/unsupported by a webserver
122 # (e.g. extractor/scivee.py, extractor/bambuser.py).
123 # However, for a test we still would like to download just a piece of a file.
124 # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
125 # block size when downloading a file.
126 if is_test
and (data_len
is None or int(data_len
) > self
._TEST
_FILE
_SIZE
):
127 data_len
= self
._TEST
_FILE
_SIZE
129 if data_len
is not None:
130 data_len
= int(data_len
) + resume_len
131 min_data_len
= self
.params
.get("min_filesize", None)
132 max_data_len
= self
.params
.get("max_filesize", None)
133 if min_data_len
is not None and data_len
< min_data_len
:
134 self
.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len
, min_data_len
))
136 if max_data_len
is not None and data_len
> max_data_len
:
137 self
.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len
, max_data_len
))
140 byte_counter
= 0 + resume_len
141 block_size
= self
.params
.get('buffersize', 1024)
144 # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
145 now
= None # needed for slow_down() in the first loop run
146 before
= start
# start measuring
150 data_block
= data
.read(block_size
if not is_test
else min(block_size
, data_len
- byte_counter
))
151 byte_counter
+= len(data_block
)
153 # exit loop when download is finished
154 if len(data_block
) == 0:
157 # Open destination file just in time
160 (stream
, tmpfilename
) = sanitize_open(tmpfilename
, open_mode
)
161 assert stream
is not None
162 filename
= self
.undo_temp_name(tmpfilename
)
163 self
.report_destination(filename
)
164 except (OSError, IOError) as err
:
165 self
.report_error('unable to open for writing: %s' % str(err
))
168 if self
.params
.get('xattr_set_filesize', False) and data_len
is not None:
171 xattr
.setxattr(tmpfilename
, 'user.ytdl.filesize', str(data_len
))
172 except(OSError, IOError, ImportError) as err
:
173 self
.report_error('unable to set filesize xattr: %s' % str(err
))
176 stream
.write(data_block
)
177 except (IOError, OSError) as err
:
179 self
.report_error('unable to write data: %s' % str(err
))
183 self
.slow_down(start
, now
, byte_counter
- resume_len
)
185 # end measuring of one loop run
190 if not self
.params
.get('noresizebuffer', False):
191 block_size
= self
.best_block_size(after
- before
, len(data_block
))
196 speed
= self
.calc_speed(start
, now
, byte_counter
- resume_len
)
200 eta
= self
.calc_eta(start
, time
.time(), data_len
- resume_len
, byte_counter
- resume_len
)
202 self
._hook
_progress
({
203 'status': 'downloading',
204 'downloaded_bytes': byte_counter
,
205 'total_bytes': data_len
,
206 'tmpfilename': tmpfilename
,
207 'filename': filename
,
210 'elapsed': now
- start
,
213 if is_test
and byte_counter
== data_len
:
218 self
.report_error('Did not get any data blocks')
220 if tmpfilename
!= '-':
223 if data_len
is not None and byte_counter
!= data_len
:
224 raise ContentTooShortError(byte_counter
, int(data_len
))
225 self
.try_rename(tmpfilename
, filename
)
227 # Update file modification time
228 if self
.params
.get('updatetime', True):
229 info_dict
['filetime'] = self
.try_utime(filename
, data
.info().get('last-modified', None))
231 self
._hook
_progress
({
232 'downloaded_bytes': byte_counter
,
233 'total_bytes': byte_counter
,
234 'filename': filename
,
235 'status': 'finished',
236 'elapsed': time
.time() - start
,