5 from .common
import FileDownloader
6 from ..networking
import Request
7 from ..networking
.exceptions
import (
8 CertificateVerifyError
,
17 XAttrUnavailableError
,
24 from ..utils
.networking
import HTTPHeaderDict
27 class HttpFD(FileDownloader
):
28 def real_download(self
, filename
, info_dict
):
29 url
= info_dict
['url']
30 request_data
= info_dict
.get('request_data', None)
32 class DownloadContext(dict):
33 __getattr__
= dict.get
34 __setattr__
= dict.__setitem
__
35 __delattr__
= dict.__delitem
__
37 ctx
= DownloadContext()
38 ctx
.filename
= filename
39 ctx
.tmpfilename
= self
.temp_name(filename
)
43 headers
= HTTPHeaderDict({'Accept-Encoding': 'identity'}
, info_dict
.get('http_headers'))
45 is_test
= self
.params
.get('test', False)
46 chunk_size
= self
._TEST
_FILE
_SIZE
if is_test
else (
47 self
.params
.get('http_chunk_size')
48 or info_dict
.get('downloader_options', {}).get('http_chunk_size')
53 ctx
.block_size
= self
.params
.get('buffersize', 1024)
54 ctx
.start_time
= time
.time()
57 req_start
, req_end
, _
= parse_http_range(headers
.get('Range'))
59 if self
.params
.get('continuedl', True):
60 # Establish possible resume length
61 if os
.path
.isfile(encodeFilename(ctx
.tmpfilename
)):
62 ctx
.resume_len
= os
.path
.getsize(
63 encodeFilename(ctx
.tmpfilename
))
65 ctx
.is_resume
= ctx
.resume_len
> 0
67 class SucceedDownload(Exception):
70 class RetryDownload(Exception):
71 def __init__(self
, source_error
):
72 self
.source_error
= source_error
74 class NextFragment(Exception):
77 def establish_connection():
78 ctx
.chunk_size
= (random
.randint(int(chunk_size
* 0.95), chunk_size
)
79 if not is_test
and chunk_size
else chunk_size
)
80 if ctx
.resume_len
> 0:
81 range_start
= ctx
.resume_len
82 if req_start
is not None:
83 # offset the beginning of Range to be within request
84 range_start
+= req_start
86 self
.report_resuming_byte(ctx
.resume_len
)
88 elif req_start
is not None:
89 range_start
= req_start
90 elif ctx
.chunk_size
> 0:
97 chunk_aware_end
= range_start
+ ctx
.chunk_size
- 1
98 # we're not allowed to download outside Range
99 range_end
= chunk_aware_end
if req_end
is None else min(chunk_aware_end
, req_end
)
100 elif req_end
is not None:
101 # there's no need for chunked downloads, so download until the end of Range
106 if try_call(lambda: range_start
> range_end
):
109 raise RetryDownload(Exception(f
'Conflicting range. (start={range_start} > end={range_end})'))
111 if try_call(lambda: range_end
>= ctx
.content_len
):
112 range_end
= ctx
.content_len
- 1
114 request
= Request(url
, request_data
, headers
)
115 has_range
= range_start
is not None
117 request
.headers
['Range'] = f
'bytes={int(range_start)}-{int_or_none(range_end) or ""}'
118 # Establish connection
120 ctx
.data
= self
.ydl
.urlopen(request
)
121 # When trying to resume, Content-Range HTTP header of response has to be checked
122 # to match the value of requested Range HTTP header. This is due to a webservers
123 # that don't support resuming and serve a whole file with no Content-Range
124 # set in response despite of requested Range (see
125 # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
127 content_range
= ctx
.data
.headers
.get('Content-Range')
128 content_range_start
, content_range_end
, content_len
= parse_http_range(content_range
)
129 # Content-Range is present and matches requested Range, resume is possible
130 if range_start
== content_range_start
and (
131 # Non-chunked download
133 # Chunked download and requested piece or
134 # its part is promised to be served
135 or content_range_end
== range_end
136 or content_len
< range_end
):
137 ctx
.content_len
= content_len
138 if content_len
or req_end
:
139 ctx
.data_len
= min(content_len
or req_end
, req_end
or content_len
) - (req_start
or 0)
141 # Content-Range is either not present or invalid. Assuming remote webserver is
142 # trying to send the whole file, resume is not possible, so wiping the local file
143 # and performing entire redownload
144 elif range_start
> 0:
145 self
.report_unable_to_resume()
148 ctx
.data_len
= ctx
.content_len
= int_or_none(ctx
.data
.headers
.get('Content-length', None))
149 except HTTPError
as err
:
150 if err
.status
== 416:
151 # Unable to resume (requested range not satisfiable)
153 # Open the connection again without the range header
154 ctx
.data
= self
.ydl
.urlopen(
155 Request(url
, request_data
, headers
))
156 content_length
= ctx
.data
.headers
['Content-Length']
157 except HTTPError
as err
:
158 if err
.status
< 500 or err
.status
>= 600:
161 # Examine the reported length
162 if (content_length
is not None
163 and (ctx
.resume_len
- 100 < int(content_length
) < ctx
.resume_len
+ 100)):
164 # The file had already been fully downloaded.
165 # Explanation to the above condition: in issue #175 it was revealed that
166 # YouTube sometimes adds or removes a few bytes from the end of the file,
167 # changing the file size slightly and causing problems for some users. So
168 # I decided to implement a suggested change and consider the file
169 # completely downloaded if the file size differs less than 100 bytes from
170 # the one in the hard drive.
171 self
.report_file_already_downloaded(ctx
.filename
)
172 self
.try_rename(ctx
.tmpfilename
, ctx
.filename
)
173 self
._hook
_progress
({
174 'filename': ctx
.filename
,
175 'status': 'finished',
176 'downloaded_bytes': ctx
.resume_len
,
177 'total_bytes': ctx
.resume_len
,
179 raise SucceedDownload
181 # The length does not match, we start the download over
182 self
.report_unable_to_resume()
186 elif err
.status
< 500 or err
.status
>= 600:
187 # Unexpected HTTP error
189 raise RetryDownload(err
)
190 except CertificateVerifyError
:
192 except TransportError
as err
:
193 raise RetryDownload(err
)
196 if ctx
.stream
is not None:
197 if ctx
.tmpfilename
!= '-':
202 data_len
= ctx
.data
.headers
.get('Content-length')
204 if ctx
.data
.headers
.get('Content-encoding'):
205 # Content-encoding is present, Content-length is not reliable anymore as we are
206 # doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
209 # Range HTTP header may be ignored/unsupported by a webserver
210 # (e.g. extractor/scivee.py, extractor/bambuser.py).
211 # However, for a test we still would like to download just a piece of a file.
212 # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
213 # block size when downloading a file.
214 if is_test
and (data_len
is None or int(data_len
) > self
._TEST
_FILE
_SIZE
):
215 data_len
= self
._TEST
_FILE
_SIZE
217 if data_len
is not None:
218 data_len
= int(data_len
) + ctx
.resume_len
219 min_data_len
= self
.params
.get('min_filesize')
220 max_data_len
= self
.params
.get('max_filesize')
221 if min_data_len
is not None and data_len
< min_data_len
:
223 f
'\r[download] File is smaller than min-filesize ({data_len} bytes < {min_data_len} bytes). Aborting.')
225 if max_data_len
is not None and data_len
> max_data_len
:
227 f
'\r[download] File is larger than max-filesize ({data_len} bytes > {max_data_len} bytes). Aborting.')
230 byte_counter
= 0 + ctx
.resume_len
231 block_size
= ctx
.block_size
234 # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
235 now
= None # needed for slow_down() in the first loop run
236 before
= start
# start measuring
240 if ctx
.tmpfilename
== '-':
241 ctx
.resume_len
= byte_counter
244 ctx
.resume_len
= os
.path
.getsize(encodeFilename(ctx
.tmpfilename
))
245 except FileNotFoundError
:
247 raise RetryDownload(e
)
252 data_block
= ctx
.data
.read(block_size
if not is_test
else min(block_size
, data_len
- byte_counter
))
253 except TransportError
as err
:
256 byte_counter
+= len(data_block
)
258 # exit loop when download is finished
259 if len(data_block
) == 0:
262 # Open destination file just in time
263 if ctx
.stream
is None:
265 ctx
.stream
, ctx
.tmpfilename
= self
.sanitize_open(
266 ctx
.tmpfilename
, ctx
.open_mode
)
267 assert ctx
.stream
is not None
268 ctx
.filename
= self
.undo_temp_name(ctx
.tmpfilename
)
269 self
.report_destination(ctx
.filename
)
270 except OSError as err
:
271 self
.report_error(f
'unable to open for writing: {err}')
274 if self
.params
.get('xattr_set_filesize', False) and data_len
is not None:
276 write_xattr(ctx
.tmpfilename
, 'user.ytdl.filesize', str(data_len
).encode())
277 except (XAttrUnavailableError
, XAttrMetadataError
) as err
:
278 self
.report_error(f
'unable to set filesize xattr: {err}')
281 ctx
.stream
.write(data_block
)
282 except OSError as err
:
284 self
.report_error(f
'unable to write data: {err}')
288 self
.slow_down(start
, now
, byte_counter
- ctx
.resume_len
)
290 # end measuring of one loop run
295 if not self
.params
.get('noresizebuffer', False):
296 block_size
= self
.best_block_size(after
- before
, len(data_block
))
301 speed
= self
.calc_speed(start
, now
, byte_counter
- ctx
.resume_len
)
302 if ctx
.data_len
is None:
305 eta
= self
.calc_eta(start
, time
.time(), ctx
.data_len
- ctx
.resume_len
, byte_counter
- ctx
.resume_len
)
307 self
._hook
_progress
({
308 'status': 'downloading',
309 'downloaded_bytes': byte_counter
,
310 'total_bytes': ctx
.data_len
,
311 'tmpfilename': ctx
.tmpfilename
,
312 'filename': ctx
.filename
,
315 'elapsed': now
- ctx
.start_time
,
316 'ctx_id': info_dict
.get('ctx_id'),
319 if data_len
is not None and byte_counter
== data_len
:
322 if speed
and speed
< (self
.params
.get('throttledratelimit') or 0):
323 # The speed must stay below the limit for 3 seconds
324 # This prevents raising error when the speed temporarily goes down
325 if ctx
.throttle_start
is None:
326 ctx
.throttle_start
= now
327 elif now
- ctx
.throttle_start
> 3:
328 if ctx
.stream
is not None and ctx
.tmpfilename
!= '-':
330 raise ThrottledDownload
332 ctx
.throttle_start
= None
334 if ctx
.stream
is None:
336 self
.report_error('Did not get any data blocks')
339 if not is_test
and ctx
.chunk_size
and ctx
.content_len
is not None and byte_counter
< ctx
.content_len
:
340 ctx
.resume_len
= byte_counter
343 if ctx
.tmpfilename
!= '-':
346 if data_len
is not None and byte_counter
!= data_len
:
347 err
= ContentTooShortError(byte_counter
, int(data_len
))
350 self
.try_rename(ctx
.tmpfilename
, ctx
.filename
)
352 # Update file modification time
353 if self
.params
.get('updatetime', True):
354 info_dict
['filetime'] = self
.try_utime(ctx
.filename
, ctx
.data
.headers
.get('last-modified', None))
356 self
._hook
_progress
({
357 'downloaded_bytes': byte_counter
,
358 'total_bytes': byte_counter
,
359 'filename': ctx
.filename
,
360 'status': 'finished',
361 'elapsed': time
.time() - ctx
.start_time
,
362 'ctx_id': info_dict
.get('ctx_id'),
367 for retry
in RetryManager(self
.params
.get('retries'), self
.report_retry
):
369 establish_connection()
371 except RetryDownload
as err
:
372 retry
.error
= err
.source_error
378 except SucceedDownload
: