1 from __future__
import unicode_literals
8 from .common
import FileDownloader
24 XAttrUnavailableError
,
27 RESPONSE_READ_EXCEPTIONS
= (TimeoutError
, ConnectionError
, ssl
.SSLError
, compat_http_client
.HTTPException
)
30 class HttpFD(FileDownloader
):
31 def real_download(self
, filename
, info_dict
):
32 url
= info_dict
['url']
33 request_data
= info_dict
.get('request_data', None)
35 class DownloadContext(dict):
36 __getattr__
= dict.get
37 __setattr__
= dict.__setitem
__
38 __delattr__
= dict.__delitem
__
40 ctx
= DownloadContext()
41 ctx
.filename
= filename
42 ctx
.tmpfilename
= self
.temp_name(filename
)
45 # Do not include the Accept-Encoding header
46 headers
= {'Youtubedl-no-compression': 'True'}
47 add_headers
= info_dict
.get('http_headers')
49 headers
.update(add_headers
)
51 is_test
= self
.params
.get('test', False)
52 chunk_size
= self
._TEST
_FILE
_SIZE
if is_test
else (
53 self
.params
.get('http_chunk_size')
54 or info_dict
.get('downloader_options', {}).get('http_chunk_size')
59 ctx
.block_size
= self
.params
.get('buffersize', 1024)
60 ctx
.start_time
= time
.time()
65 req_start
, req_end
, _
= parse_http_range(headers
.get('Range'))
67 if self
.params
.get('continuedl', True):
68 # Establish possible resume length
69 if os
.path
.isfile(encodeFilename(ctx
.tmpfilename
)):
70 ctx
.resume_len
= os
.path
.getsize(
71 encodeFilename(ctx
.tmpfilename
))
73 ctx
.is_resume
= ctx
.resume_len
> 0
76 retries
= self
.params
.get('retries', 0)
78 class SucceedDownload(Exception):
81 class RetryDownload(Exception):
82 def __init__(self
, source_error
):
83 self
.source_error
= source_error
85 class NextFragment(Exception):
88 def set_range(req
, start
, end
):
89 range_header
= 'bytes=%d-' % start
91 range_header
+= compat_str(end
)
92 req
.add_header('Range', range_header
)
94 def establish_connection():
95 ctx
.chunk_size
= (random
.randint(int(chunk_size
* 0.95), chunk_size
)
96 if not is_test
and chunk_size
else chunk_size
)
97 if ctx
.resume_len
> 0:
98 range_start
= ctx
.resume_len
99 if req_start
is not None:
100 # offset the beginning of Range to be within request
101 range_start
+= req_start
103 self
.report_resuming_byte(ctx
.resume_len
)
105 elif req_start
is not None:
106 range_start
= req_start
107 elif ctx
.chunk_size
> 0:
111 ctx
.is_resume
= False
114 chunk_aware_end
= range_start
+ ctx
.chunk_size
- 1
115 # we're not allowed to download outside Range
116 range_end
= chunk_aware_end
if req_end
is None else min(chunk_aware_end
, req_end
)
117 elif req_end
is not None:
118 # there's no need for chunked downloads, so download until the end of Range
123 if try_call(lambda: range_start
> range_end
):
126 raise RetryDownload(Exception(f
'Conflicting range. (start={range_start} > end={range_end})'))
128 if try_call(lambda: range_end
>= ctx
.content_len
):
129 range_end
= ctx
.content_len
- 1
131 request
= sanitized_Request(url
, request_data
, headers
)
132 has_range
= range_start
is not None
134 set_range(request
, range_start
, range_end
)
135 # Establish connection
137 ctx
.data
= self
.ydl
.urlopen(request
)
138 # When trying to resume, Content-Range HTTP header of response has to be checked
139 # to match the value of requested Range HTTP header. This is due to a webservers
140 # that don't support resuming and serve a whole file with no Content-Range
141 # set in response despite of requested Range (see
142 # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
144 content_range
= ctx
.data
.headers
.get('Content-Range')
145 content_range_start
, content_range_end
, content_len
= parse_http_range(content_range
)
146 if content_range_start
is not None and range_start
== content_range_start
:
147 # Content-Range is present and matches requested Range, resume is possible
148 accept_content_len
= (
149 # Non-chunked download
151 # Chunked download and requested piece or
152 # its part is promised to be served
153 or content_range_end
== range_end
154 or content_len
< range_end
)
155 if accept_content_len
:
156 ctx
.content_len
= content_len
157 ctx
.data_len
= min(content_len
, req_end
or content_len
) - (req_start
or 0)
159 # Content-Range is either not present or invalid. Assuming remote webserver is
160 # trying to send the whole file, resume is not possible, so wiping the local file
161 # and performing entire redownload
162 self
.report_unable_to_resume()
165 ctx
.data_len
= ctx
.content_len
= int_or_none(ctx
.data
.info().get('Content-length', None))
166 except (compat_urllib_error
.HTTPError
, ) as err
:
168 # Unable to resume (requested range not satisfiable)
170 # Open the connection again without the range header
171 ctx
.data
= self
.ydl
.urlopen(
172 sanitized_Request(url
, request_data
, headers
))
173 content_length
= ctx
.data
.info()['Content-Length']
174 except (compat_urllib_error
.HTTPError
, ) as err
:
175 if err
.code
< 500 or err
.code
>= 600:
178 # Examine the reported length
179 if (content_length
is not None
180 and (ctx
.resume_len
- 100 < int(content_length
) < ctx
.resume_len
+ 100)):
181 # The file had already been fully downloaded.
182 # Explanation to the above condition: in issue #175 it was revealed that
183 # YouTube sometimes adds or removes a few bytes from the end of the file,
184 # changing the file size slightly and causing problems for some users. So
185 # I decided to implement a suggested change and consider the file
186 # completely downloaded if the file size differs less than 100 bytes from
187 # the one in the hard drive.
188 self
.report_file_already_downloaded(ctx
.filename
)
189 self
.try_rename(ctx
.tmpfilename
, ctx
.filename
)
190 self
._hook
_progress
({
191 'filename': ctx
.filename
,
192 'status': 'finished',
193 'downloaded_bytes': ctx
.resume_len
,
194 'total_bytes': ctx
.resume_len
,
196 raise SucceedDownload()
198 # The length does not match, we start the download over
199 self
.report_unable_to_resume()
203 elif err
.code
< 500 or err
.code
>= 600:
204 # Unexpected HTTP error
206 raise RetryDownload(err
)
207 except compat_urllib_error
.URLError
as err
:
208 if isinstance(err
.reason
, ssl
.CertificateError
):
210 raise RetryDownload(err
)
211 # In urllib.request.AbstractHTTPHandler, the response is partially read on request.
212 # Any errors that occur during this will not be wrapped by URLError
213 except RESPONSE_READ_EXCEPTIONS
as err
:
214 raise RetryDownload(err
)
217 nonlocal throttle_start
218 data_len
= ctx
.data
.info().get('Content-length', None)
220 # Range HTTP header may be ignored/unsupported by a webserver
221 # (e.g. extractor/scivee.py, extractor/bambuser.py).
222 # However, for a test we still would like to download just a piece of a file.
223 # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
224 # block size when downloading a file.
225 if is_test
and (data_len
is None or int(data_len
) > self
._TEST
_FILE
_SIZE
):
226 data_len
= self
._TEST
_FILE
_SIZE
228 if data_len
is not None:
229 data_len
= int(data_len
) + ctx
.resume_len
230 min_data_len
= self
.params
.get('min_filesize')
231 max_data_len
= self
.params
.get('max_filesize')
232 if min_data_len
is not None and data_len
< min_data_len
:
233 self
.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len
, min_data_len
))
235 if max_data_len
is not None and data_len
> max_data_len
:
236 self
.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len
, max_data_len
))
239 byte_counter
= 0 + ctx
.resume_len
240 block_size
= ctx
.block_size
243 # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
244 now
= None # needed for slow_down() in the first loop run
245 before
= start
# start measuring
248 to_stdout
= ctx
.tmpfilename
== '-'
249 if ctx
.stream
is not None:
253 ctx
.resume_len
= byte_counter
if to_stdout
else os
.path
.getsize(encodeFilename(ctx
.tmpfilename
))
254 raise RetryDownload(e
)
259 data_block
= ctx
.data
.read(block_size
if not is_test
else min(block_size
, data_len
- byte_counter
))
260 except RESPONSE_READ_EXCEPTIONS
as err
:
263 byte_counter
+= len(data_block
)
265 # exit loop when download is finished
266 if len(data_block
) == 0:
269 # Open destination file just in time
270 if ctx
.stream
is None:
272 ctx
.stream
, ctx
.tmpfilename
= self
.sanitize_open(
273 ctx
.tmpfilename
, ctx
.open_mode
)
274 assert ctx
.stream
is not None
275 ctx
.filename
= self
.undo_temp_name(ctx
.tmpfilename
)
276 self
.report_destination(ctx
.filename
)
277 except (OSError, IOError) as err
:
278 self
.report_error('unable to open for writing: %s' % str(err
))
281 if self
.params
.get('xattr_set_filesize', False) and data_len
is not None:
283 write_xattr(ctx
.tmpfilename
, 'user.ytdl.filesize', str(data_len
).encode('utf-8'))
284 except (XAttrUnavailableError
, XAttrMetadataError
) as err
:
285 self
.report_error('unable to set filesize xattr: %s' % str(err
))
288 ctx
.stream
.write(data_block
)
289 except (IOError, OSError) as err
:
291 self
.report_error('unable to write data: %s' % str(err
))
295 self
.slow_down(start
, now
, byte_counter
- ctx
.resume_len
)
297 # end measuring of one loop run
302 if not self
.params
.get('noresizebuffer', False):
303 block_size
= self
.best_block_size(after
- before
, len(data_block
))
308 speed
= self
.calc_speed(start
, now
, byte_counter
- ctx
.resume_len
)
309 if ctx
.data_len
is None:
312 eta
= self
.calc_eta(start
, time
.time(), ctx
.data_len
- ctx
.resume_len
, byte_counter
- ctx
.resume_len
)
314 self
._hook
_progress
({
315 'status': 'downloading',
316 'downloaded_bytes': byte_counter
,
317 'total_bytes': ctx
.data_len
,
318 'tmpfilename': ctx
.tmpfilename
,
319 'filename': ctx
.filename
,
322 'elapsed': now
- ctx
.start_time
,
323 'ctx_id': info_dict
.get('ctx_id'),
326 if data_len
is not None and byte_counter
== data_len
:
329 if speed
and speed
< (self
.params
.get('throttledratelimit') or 0):
330 # The speed must stay below the limit for 3 seconds
331 # This prevents raising error when the speed temporarily goes down
332 if throttle_start
is None:
334 elif now
- throttle_start
> 3:
335 if ctx
.stream
is not None and ctx
.tmpfilename
!= '-':
337 raise ThrottledDownload()
339 throttle_start
= None
341 if not is_test
and ctx
.chunk_size
and ctx
.content_len
is not None and byte_counter
< ctx
.content_len
:
342 ctx
.resume_len
= byte_counter
343 # ctx.block_size = block_size
346 if ctx
.stream
is None:
348 self
.report_error('Did not get any data blocks')
350 if ctx
.tmpfilename
!= '-':
353 if data_len
is not None and byte_counter
!= data_len
:
354 err
= ContentTooShortError(byte_counter
, int(data_len
))
359 self
.try_rename(ctx
.tmpfilename
, ctx
.filename
)
361 # Update file modification time
362 if self
.params
.get('updatetime', True):
363 info_dict
['filetime'] = self
.try_utime(ctx
.filename
, ctx
.data
.info().get('last-modified', None))
365 self
._hook
_progress
({
366 'downloaded_bytes': byte_counter
,
367 'total_bytes': byte_counter
,
368 'filename': ctx
.filename
,
369 'status': 'finished',
370 'elapsed': time
.time() - ctx
.start_time
,
371 'ctx_id': info_dict
.get('ctx_id'),
376 while count
<= retries
:
378 establish_connection()
380 except RetryDownload
as e
:
383 self
.report_retry(e
.source_error
, count
, retries
)
385 self
.to_screen(f
'[download] Got server HTTP error: {e.source_error}')
389 except SucceedDownload
:
392 self
.report_error('giving up after %s retries' % retries
)