1 from __future__
import unicode_literals
10 from .common
import FileDownloader
11 from ..compat
import (
23 XAttrUnavailableError
,
27 class HttpFD(FileDownloader
):
28 def real_download(self
, filename
, info_dict
):
29 url
= info_dict
['url']
30 request_data
= info_dict
.get('request_data', None)
32 class DownloadContext(dict):
33 __getattr__
= dict.get
34 __setattr__
= dict.__setitem
__
35 __delattr__
= dict.__delitem
__
37 ctx
= DownloadContext()
38 ctx
.filename
= filename
39 ctx
.tmpfilename
= self
.temp_name(filename
)
42 # Do not include the Accept-Encoding header
43 headers
= {'Youtubedl-no-compression': 'True'}
44 add_headers
= info_dict
.get('http_headers')
46 headers
.update(add_headers
)
48 is_test
= self
.params
.get('test', False)
49 chunk_size
= self
._TEST
_FILE
_SIZE
if is_test
else (
50 info_dict
.get('downloader_options', {}).get('http_chunk_size')
51 or self
.params
.get('http_chunk_size') or 0)
56 ctx
.block_size
= self
.params
.get('buffersize', 1024)
57 ctx
.start_time
= time
.time()
60 if self
.params
.get('continuedl', True):
61 # Establish possible resume length
62 if os
.path
.isfile(encodeFilename(ctx
.tmpfilename
)):
63 ctx
.resume_len
= os
.path
.getsize(
64 encodeFilename(ctx
.tmpfilename
))
66 ctx
.is_resume
= ctx
.resume_len
> 0
69 retries
= self
.params
.get('retries', 0)
71 class SucceedDownload(Exception):
74 class RetryDownload(Exception):
75 def __init__(self
, source_error
):
76 self
.source_error
= source_error
78 class NextFragment(Exception):
81 def set_range(req
, start
, end
):
82 range_header
= 'bytes=%d-' % start
84 range_header
+= compat_str(end
)
85 req
.add_header('Range', range_header
)
87 def establish_connection():
88 ctx
.chunk_size
= (random
.randint(int(chunk_size
* 0.95), chunk_size
)
89 if not is_test
and chunk_size
else chunk_size
)
90 if ctx
.resume_len
> 0:
91 range_start
= ctx
.resume_len
93 self
.report_resuming_byte(ctx
.resume_len
)
95 elif ctx
.chunk_size
> 0:
100 range_end
= range_start
+ ctx
.chunk_size
- 1 if ctx
.chunk_size
else None
101 if range_end
and ctx
.data_len
is not None and range_end
>= ctx
.data_len
:
102 range_end
= ctx
.data_len
- 1
103 has_range
= range_start
is not None
104 ctx
.has_range
= has_range
105 request
= sanitized_Request(url
, request_data
, headers
)
107 set_range(request
, range_start
, range_end
)
108 # Establish connection
111 ctx
.data
= self
.ydl
.urlopen(request
)
112 except (compat_urllib_error
.URLError
, ) as err
:
113 # reason may not be available, e.g. for urllib2.HTTPError on python 2.6
114 reason
= getattr(err
, 'reason', None)
115 if isinstance(reason
, socket
.timeout
):
116 raise RetryDownload(err
)
118 # When trying to resume, Content-Range HTTP header of response has to be checked
119 # to match the value of requested Range HTTP header. This is due to a webservers
120 # that don't support resuming and serve a whole file with no Content-Range
121 # set in response despite of requested Range (see
122 # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
124 content_range
= ctx
.data
.headers
.get('Content-Range')
126 content_range_m
= re
.search(r
'bytes (\d+)-(\d+)?(?:/(\d+))?', content_range
)
127 # Content-Range is present and matches requested Range, resume is possible
129 if range_start
== int(content_range_m
.group(1)):
130 content_range_end
= int_or_none(content_range_m
.group(2))
131 content_len
= int_or_none(content_range_m
.group(3))
132 accept_content_len
= (
133 # Non-chunked download
135 # Chunked download and requested piece or
136 # its part is promised to be served
137 or content_range_end
== range_end
138 or content_len
< range_end
)
139 if accept_content_len
:
140 ctx
.data_len
= content_len
142 # Content-Range is either not present or invalid. Assuming remote webserver is
143 # trying to send the whole file, resume is not possible, so wiping the local file
144 # and performing entire redownload
145 self
.report_unable_to_resume()
148 ctx
.data_len
= int_or_none(ctx
.data
.info().get('Content-length', None))
150 except (compat_urllib_error
.HTTPError
, ) as err
:
152 # Unable to resume (requested range not satisfiable)
154 # Open the connection again without the range header
155 ctx
.data
= self
.ydl
.urlopen(
156 sanitized_Request(url
, request_data
, headers
))
157 content_length
= ctx
.data
.info()['Content-Length']
158 except (compat_urllib_error
.HTTPError
, ) as err
:
159 if err
.code
< 500 or err
.code
>= 600:
162 # Examine the reported length
163 if (content_length
is not None
164 and (ctx
.resume_len
- 100 < int(content_length
) < ctx
.resume_len
+ 100)):
165 # The file had already been fully downloaded.
166 # Explanation to the above condition: in issue #175 it was revealed that
167 # YouTube sometimes adds or removes a few bytes from the end of the file,
168 # changing the file size slightly and causing problems for some users. So
169 # I decided to implement a suggested change and consider the file
170 # completely downloaded if the file size differs less than 100 bytes from
171 # the one in the hard drive.
172 self
.report_file_already_downloaded(ctx
.filename
)
173 self
.try_rename(ctx
.tmpfilename
, ctx
.filename
)
174 self
._hook
_progress
({
175 'filename': ctx
.filename
,
176 'status': 'finished',
177 'downloaded_bytes': ctx
.resume_len
,
178 'total_bytes': ctx
.resume_len
,
180 raise SucceedDownload()
182 # The length does not match, we start the download over
183 self
.report_unable_to_resume()
187 elif err
.code
< 500 or err
.code
>= 600:
188 # Unexpected HTTP error
190 raise RetryDownload(err
)
191 except socket
.error
as err
:
192 if err
.errno
!= errno
.ECONNRESET
:
193 # Connection reset is no problem, just retry
195 raise RetryDownload(err
)
198 data_len
= ctx
.data
.info().get('Content-length', None)
200 # Range HTTP header may be ignored/unsupported by a webserver
201 # (e.g. extractor/scivee.py, extractor/bambuser.py).
202 # However, for a test we still would like to download just a piece of a file.
203 # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
204 # block size when downloading a file.
205 if is_test
and (data_len
is None or int(data_len
) > self
._TEST
_FILE
_SIZE
):
206 data_len
= self
._TEST
_FILE
_SIZE
208 if data_len
is not None:
209 data_len
= int(data_len
) + ctx
.resume_len
210 min_data_len
= self
.params
.get('min_filesize')
211 max_data_len
= self
.params
.get('max_filesize')
212 if min_data_len
is not None and data_len
< min_data_len
:
213 self
.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len
, min_data_len
))
215 if max_data_len
is not None and data_len
> max_data_len
:
216 self
.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len
, max_data_len
))
219 byte_counter
= 0 + ctx
.resume_len
220 block_size
= ctx
.block_size
223 # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
224 now
= None # needed for slow_down() in the first loop run
225 before
= start
# start measuring
228 to_stdout
= ctx
.tmpfilename
== '-'
229 if ctx
.stream
is not None:
233 ctx
.resume_len
= byte_counter
if to_stdout
else os
.path
.getsize(encodeFilename(ctx
.tmpfilename
))
234 raise RetryDownload(e
)
239 data_block
= ctx
.data
.read(block_size
if data_len
is None else min(block_size
, data_len
- byte_counter
))
240 # socket.timeout is a subclass of socket.error but may not have
242 except socket
.timeout
as e
:
244 except socket
.error
as e
:
245 # SSLError on python 2 (inherits socket.error) may have
246 # no errno set but this error message
247 if e
.errno
in (errno
.ECONNRESET
, errno
.ETIMEDOUT
) or getattr(e
, 'message', None) == 'The read operation timed out':
251 byte_counter
+= len(data_block
)
253 # exit loop when download is finished
254 if len(data_block
) == 0:
257 # Open destination file just in time
258 if ctx
.stream
is None:
260 ctx
.stream
, ctx
.tmpfilename
= sanitize_open(
261 ctx
.tmpfilename
, ctx
.open_mode
)
262 assert ctx
.stream
is not None
263 ctx
.filename
= self
.undo_temp_name(ctx
.tmpfilename
)
264 self
.report_destination(ctx
.filename
)
265 except (OSError, IOError) as err
:
266 self
.report_error('unable to open for writing: %s' % str(err
))
269 if self
.params
.get('xattr_set_filesize', False) and data_len
is not None:
271 write_xattr(ctx
.tmpfilename
, 'user.ytdl.filesize', str(data_len
).encode('utf-8'))
272 except (XAttrUnavailableError
, XAttrMetadataError
) as err
:
273 self
.report_error('unable to set filesize xattr: %s' % str(err
))
276 ctx
.stream
.write(data_block
)
277 except (IOError, OSError) as err
:
279 self
.report_error('unable to write data: %s' % str(err
))
283 self
.slow_down(start
, now
, byte_counter
- ctx
.resume_len
)
285 # end measuring of one loop run
290 if not self
.params
.get('noresizebuffer', False):
291 block_size
= self
.best_block_size(after
- before
, len(data_block
))
296 speed
= self
.calc_speed(start
, now
, byte_counter
- ctx
.resume_len
)
297 if ctx
.data_len
is None:
300 eta
= self
.calc_eta(start
, time
.time(), ctx
.data_len
- ctx
.resume_len
, byte_counter
- ctx
.resume_len
)
302 self
._hook
_progress
({
303 'status': 'downloading',
304 'downloaded_bytes': byte_counter
,
305 'total_bytes': ctx
.data_len
,
306 'tmpfilename': ctx
.tmpfilename
,
307 'filename': ctx
.filename
,
310 'elapsed': now
- ctx
.start_time
,
313 if data_len
is not None and byte_counter
== data_len
:
316 if not is_test
and ctx
.chunk_size
and ctx
.data_len
is not None and byte_counter
< ctx
.data_len
:
317 ctx
.resume_len
= byte_counter
318 # ctx.block_size = block_size
321 if ctx
.stream
is None:
323 self
.report_error('Did not get any data blocks')
325 if ctx
.tmpfilename
!= '-':
328 if data_len
is not None and byte_counter
!= data_len
:
329 err
= ContentTooShortError(byte_counter
, int(data_len
))
334 self
.try_rename(ctx
.tmpfilename
, ctx
.filename
)
336 # Update file modification time
337 if self
.params
.get('updatetime', True):
338 info_dict
['filetime'] = self
.try_utime(ctx
.filename
, ctx
.data
.info().get('last-modified', None))
340 self
._hook
_progress
({
341 'downloaded_bytes': byte_counter
,
342 'total_bytes': byte_counter
,
343 'filename': ctx
.filename
,
344 'status': 'finished',
345 'elapsed': time
.time() - ctx
.start_time
,
350 while count
<= retries
:
352 establish_connection()
354 except RetryDownload
as e
:
357 self
.report_retry(e
.source_error
, count
, retries
)
361 except SucceedDownload
:
364 self
.report_error('giving up after %s retries' % retries
)