1 from __future__
import division
, unicode_literals
17 from ..minicurses
import (
20 QuietMultilinePrinter
,
21 BreaklineStatusPrinter
25 class FileDownloader(object):
26 """File Downloader class.
28 File downloader objects are the ones responsible of downloading the
29 actual video file and writing it to disk.
31 File downloaders accept a lot of parameters. In order not to saturate
32 the object constructor with arguments, it receives a dictionary of
37 verbose: Print additional info to stdout.
38 quiet: Do not print messages to stdout.
39 ratelimit: Download speed limit, in bytes/sec.
40 throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
41 retries: Number of times to retry for HTTP error 5xx
42 buffersize: Size of download buffer in bytes.
43 noresizebuffer: Do not automatically resize the download buffer.
44 continuedl: Try to continue downloads if possible.
45 noprogress: Do not print the progress bar.
46 nopart: Do not use temporary .part files.
47 updatetime: Use the Last-modified header to set output file timestamps.
48 test: Download only first bytes to test the downloader.
49 min_filesize: Skip files smaller than this size
50 max_filesize: Skip files larger than this size
51 xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
52 external_downloader_args: A dictionary of downloader keys (in lower case)
53 and a list of additional command-line arguments for the
54 executable. Use 'default' as the name for arguments to be
55 passed to all downloaders. For compatibility with youtube-dl,
56 a single list of args can also be used
57 hls_use_mpegts: Use the mpegts container for HLS videos.
58 http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be
59 useful for bypassing bandwidth throttling imposed by
60 a webserver (experimental)
61 progress_template: See YoutubeDL.py
63 Subclasses of this one must re-define the real_download method.
66 _TEST_FILE_SIZE
= 10241
69 def __init__(self
, ydl
, params
):
70 """Create a FileDownloader object with the given options."""
72 self
._progress
_hooks
= []
74 self
._prepare
_multiline
_status
()
75 self
.add_progress_hook(self
.report_progress
)
78 def format_seconds(seconds
):
79 time
= timetuple_from_msec(seconds
* 1000)
83 return '%02d:%02d' % time
[1:-1]
84 return '%02d:%02d:%02d' % time
[:-1]
87 def calc_percent(byte_counter
, data_len
):
90 return float(byte_counter
) / float(data_len
) * 100.0
93 def format_percent(percent
):
98 return '%6s' % ('%3.1f%%' % percent
)
101 def calc_eta(start
, now
, total
, current
):
107 if current
== 0 or dif
< 0.001: # One millisecond
109 rate
= float(current
) / dif
110 return int((float(total
) - float(current
)) / rate
)
116 return FileDownloader
.format_seconds(eta
)
119 def calc_speed(start
, now
, bytes):
121 if bytes == 0 or dif
< 0.001: # One millisecond
123 return float(bytes) / dif
126 def format_speed(speed
):
128 return '%10s' % '---b/s'
129 return '%10s' % ('%s/s' % format_bytes(speed
))
132 def format_retries(retries
):
133 return 'inf' if retries
== float('inf') else '%.0f' % retries
136 def best_block_size(elapsed_time
, bytes):
137 new_min
= max(bytes / 2.0, 1.0)
138 new_max
= min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
139 if elapsed_time
< 0.001:
141 rate
= bytes / elapsed_time
149 def parse_bytes(bytestr
):
150 """Parse a string indicating a byte quantity into an integer."""
151 matchobj
= re
.match(r
'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr
)
154 number
= float(matchobj
.group(1))
155 multiplier
= 1024.0 ** 'bkmgtpezy'.index(matchobj
.group(2).lower())
156 return int(round(number
* multiplier
))
158 def to_screen(self
, *args
, **kargs
):
159 self
.ydl
.to_stdout(*args
, quiet
=self
.params
.get('quiet'), **kargs
)
161 def to_stderr(self
, message
):
162 self
.ydl
.to_stderr(message
)
164 def to_console_title(self
, message
):
165 self
.ydl
.to_console_title(message
)
167 def trouble(self
, *args
, **kargs
):
168 self
.ydl
.trouble(*args
, **kargs
)
170 def report_warning(self
, *args
, **kargs
):
171 self
.ydl
.report_warning(*args
, **kargs
)
173 def report_error(self
, *args
, **kargs
):
174 self
.ydl
.report_error(*args
, **kargs
)
176 def write_debug(self
, *args
, **kargs
):
177 self
.ydl
.write_debug(*args
, **kargs
)
179 def slow_down(self
, start_time
, now
, byte_counter
):
180 """Sleep if the download speed is over the rate limit."""
181 rate_limit
= self
.params
.get('ratelimit')
182 if rate_limit
is None or byte_counter
== 0:
186 elapsed
= now
- start_time
189 speed
= float(byte_counter
) / elapsed
190 if speed
> rate_limit
:
191 sleep_time
= float(byte_counter
) / rate_limit
- elapsed
193 time
.sleep(sleep_time
)
195 def temp_name(self
, filename
):
196 """Returns a temporary filename for the given filename."""
197 if self
.params
.get('nopart', False) or filename
== '-' or \
198 (os
.path
.exists(encodeFilename(filename
)) and not os
.path
.isfile(encodeFilename(filename
))):
200 return filename
+ '.part'
202 def undo_temp_name(self
, filename
):
203 if filename
.endswith('.part'):
204 return filename
[:-len('.part')]
207 def ytdl_filename(self
, filename
):
208 return filename
+ '.ytdl'
210 def try_rename(self
, old_filename
, new_filename
):
211 if old_filename
== new_filename
:
214 os
.replace(old_filename
, new_filename
)
215 except (IOError, OSError) as err
:
216 self
.report_error(f
'unable to rename file: {err}')
218 def try_utime(self
, filename
, last_modified_hdr
):
219 """Try to set the last-modified time of the given file."""
220 if last_modified_hdr
is None:
222 if not os
.path
.isfile(encodeFilename(filename
)):
224 timestr
= last_modified_hdr
227 filetime
= timeconvert(timestr
)
230 # Ignore obviously invalid dates
234 os
.utime(filename
, (time
.time(), filetime
))
239 def report_destination(self
, filename
):
240 """Report destination filename."""
241 self
.to_screen('[download] Destination: ' + filename
)
243 def _prepare_multiline_status(self
, lines
=1):
244 if self
.params
.get('noprogress'):
245 self
._multiline
= QuietMultilinePrinter()
246 elif self
.ydl
.params
.get('logger'):
247 self
._multiline
= MultilineLogger(self
.ydl
.params
['logger'], lines
)
248 elif self
.params
.get('progress_with_newline'):
249 self
._multiline
= BreaklineStatusPrinter(self
.ydl
._screen
_file
, lines
)
251 self
._multiline
= MultilinePrinter(self
.ydl
._screen
_file
, lines
, not self
.params
.get('quiet'))
252 self
._multiline
.allow_colors
= self
._multiline
._HAVE
_FULLCAP
and not self
.params
.get('no_color')
254 def _finish_multiline_status(self
):
255 self
._multiline
.end()
258 'downloaded_bytes': 'light blue',
259 'percent': 'light blue',
262 'elapsed': 'bold white',
264 'total_bytes_estimate': '',
267 def _report_progress_status(self
, s
, default_template
):
268 for name
, style
in self
._progress
_styles
.items():
269 name
= f
'_{name}_str'
272 s
[name
] = self
._format
_progress
(s
[name
], style
)
273 s
['_default_template'] = default_template
% s
275 progress_dict
= s
.copy()
276 progress_dict
.pop('info_dict')
277 progress_dict
= {'info': s['info_dict'], 'progress': progress_dict}
279 progress_template
= self
.params
.get('progress_template', {})
280 self
._multiline
.print_at_line(self
.ydl
.evaluate_outtmpl(
281 progress_template
.get('download') or '[download] %(progress._default_template)s',
282 progress_dict
), s
.get('progress_idx') or 0)
283 self
.to_console_title(self
.ydl
.evaluate_outtmpl(
284 progress_template
.get('download-title') or 'yt-dlp %(progress._default_template)s',
287 def _format_progress(self
, *args
, **kwargs
):
288 return self
.ydl
._format
_text
(
289 self
._multiline
.stream
, self
._multiline
.allow_colors
, *args
, **kwargs
)
291 def report_progress(self
, s
):
292 if s
['status'] == 'finished':
293 if self
.params
.get('noprogress'):
294 self
.to_screen('[download] Download completed')
295 msg_template
= '100%%'
296 if s
.get('total_bytes') is not None:
297 s
['_total_bytes_str'] = format_bytes(s
['total_bytes'])
298 msg_template
+= ' of %(_total_bytes_str)s'
299 if s
.get('elapsed') is not None:
300 s
['_elapsed_str'] = self
.format_seconds(s
['elapsed'])
301 msg_template
+= ' in %(_elapsed_str)s'
302 s
['_percent_str'] = self
.format_percent(100)
303 self
._report
_progress
_status
(s
, msg_template
)
306 if s
['status'] != 'downloading':
309 if s
.get('eta') is not None:
310 s
['_eta_str'] = self
.format_eta(s
['eta'])
312 s
['_eta_str'] = 'Unknown'
314 if s
.get('total_bytes') and s
.get('downloaded_bytes') is not None:
315 s
['_percent_str'] = self
.format_percent(100 * s
['downloaded_bytes'] / s
['total_bytes'])
316 elif s
.get('total_bytes_estimate') and s
.get('downloaded_bytes') is not None:
317 s
['_percent_str'] = self
.format_percent(100 * s
['downloaded_bytes'] / s
['total_bytes_estimate'])
319 if s
.get('downloaded_bytes') == 0:
320 s
['_percent_str'] = self
.format_percent(0)
322 s
['_percent_str'] = 'Unknown %'
324 if s
.get('speed') is not None:
325 s
['_speed_str'] = self
.format_speed(s
['speed'])
327 s
['_speed_str'] = 'Unknown speed'
329 if s
.get('total_bytes') is not None:
330 s
['_total_bytes_str'] = format_bytes(s
['total_bytes'])
331 msg_template
= '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
332 elif s
.get('total_bytes_estimate') is not None:
333 s
['_total_bytes_estimate_str'] = format_bytes(s
['total_bytes_estimate'])
334 msg_template
= '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
336 if s
.get('downloaded_bytes') is not None:
337 s
['_downloaded_bytes_str'] = format_bytes(s
['downloaded_bytes'])
339 s
['_elapsed_str'] = self
.format_seconds(s
['elapsed'])
340 msg_template
= '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
342 msg_template
= '%(_downloaded_bytes_str)s at %(_speed_str)s'
344 msg_template
= '%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s'
345 if s
.get('fragment_index') and s
.get('fragment_count'):
346 msg_template
+= ' (frag %(fragment_index)s/%(fragment_count)s)'
347 elif s
.get('fragment_index'):
348 msg_template
+= ' (frag %(fragment_index)s)'
349 self
._report
_progress
_status
(s
, msg_template
)
351 def report_resuming_byte(self
, resume_len
):
352 """Report attempt to resume at given byte."""
353 self
.to_screen('[download] Resuming download at byte %s' % resume_len
)
355 def report_retry(self
, err
, count
, retries
):
356 """Report retry in case of HTTP error 5xx"""
358 '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
359 % (error_to_compat_str(err
), count
, self
.format_retries(retries
)))
361 def report_file_already_downloaded(self
, *args
, **kwargs
):
362 """Report file has already been fully downloaded."""
363 return self
.ydl
.report_file_already_downloaded(*args
, **kwargs
)
365 def report_unable_to_resume(self
):
366 """Report it was impossible to resume download."""
367 self
.to_screen('[download] Unable to resume')
370 def supports_manifest(manifest
):
371 """ Whether the downloader can download the fragments from the manifest.
372 Redefine in subclasses if needed. """
375 def download(self
, filename
, info_dict
, subtitle
=False):
376 """Download to a filename using the info from info_dict
377 Return True on success and False otherwise
380 nooverwrites_and_exists
= (
381 not self
.params
.get('overwrites', True)
382 and os
.path
.exists(encodeFilename(filename
))
385 if not hasattr(filename
, 'write'):
386 continuedl_and_exists
= (
387 self
.params
.get('continuedl', True)
388 and os
.path
.isfile(encodeFilename(filename
))
389 and not self
.params
.get('nopart', False)
392 # Check file already present
393 if filename
!= '-' and (nooverwrites_and_exists
or continuedl_and_exists
):
394 self
.report_file_already_downloaded(filename
)
395 self
._hook
_progress
({
396 'filename': filename
,
397 'status': 'finished',
398 'total_bytes': os
.path
.getsize(encodeFilename(filename
)),
402 if subtitle
is False:
403 min_sleep_interval
= self
.params
.get('sleep_interval')
404 if min_sleep_interval
:
405 max_sleep_interval
= self
.params
.get('max_sleep_interval', min_sleep_interval
)
406 sleep_interval
= random
.uniform(min_sleep_interval
, max_sleep_interval
)
408 '[download] Sleeping %s seconds ...' % (
409 int(sleep_interval
) if sleep_interval
.is_integer()
410 else '%.2f' % sleep_interval
))
411 time
.sleep(sleep_interval
)
413 sleep_interval_sub
= 0
414 if type(self
.params
.get('sleep_interval_subtitles')) is int:
415 sleep_interval_sub
= self
.params
.get('sleep_interval_subtitles')
416 if sleep_interval_sub
> 0:
418 '[download] Sleeping %s seconds ...' % (
420 time
.sleep(sleep_interval_sub
)
421 ret
= self
.real_download(filename
, info_dict
)
422 self
._finish
_multiline
_status
()
425 def real_download(self
, filename
, info_dict
):
426 """Real download process. Redefine in subclasses."""
427 raise NotImplementedError('This method must be implemented by subclasses')
429 def _hook_progress(self
, status
, info_dict
):
430 if not self
._progress
_hooks
:
432 status
['info_dict'] = info_dict
433 # youtube-dl passes the same status object to all the hooks.
434 # Some third party scripts seems to be relying on this.
435 # So keep this behavior if possible
436 for ph
in self
._progress
_hooks
:
439 def add_progress_hook(self
, ph
):
440 # See YoutubeDl.py (search for progress_hooks) for a description of
442 self
._progress
_hooks
.append(ph
)
444 def _debug_cmd(self
, args
, exe
=None):
445 if not self
.params
.get('verbose', False):
448 str_args
= [decodeArgument(a
) for a
in args
]
451 exe
= os
.path
.basename(str_args
[0])
453 self
.write_debug('%s command line: %s' % (exe
, shell_quote(str_args
)))