]>
Commit | Line | Data |
---|---|---|
5cda4eda | 1 | from __future__ import division, unicode_literals |
b6b70730 | 2 | |
3bc2ddcc JMF |
3 | import os |
4 | import re | |
3bc2ddcc JMF |
5 | import sys |
6 | import time | |
065bc354 | 7 | import random |
3bc2ddcc | 8 | |
e9c0cdd3 | 9 | from ..compat import compat_os_name |
3bc2ddcc | 10 | from ..utils import ( |
1433734c | 11 | decodeArgument, |
3bc2ddcc | 12 | encodeFilename, |
9b9c5355 | 13 | error_to_compat_str, |
3bc2ddcc | 14 | format_bytes, |
1433734c | 15 | shell_quote, |
e3ced9ed | 16 | timeconvert, |
3bc2ddcc JMF |
17 | ) |
18 | ||
19 | ||
20 | class FileDownloader(object): | |
21 | """File Downloader class. | |
22 | ||
23 | File downloader objects are the ones responsible of downloading the | |
24 | actual video file and writing it to disk. | |
25 | ||
26 | File downloaders accept a lot of parameters. In order not to saturate | |
27 | the object constructor with arguments, it receives a dictionary of | |
28 | options instead. | |
29 | ||
30 | Available options: | |
31 | ||
881e6a1f PH |
32 | verbose: Print additional info to stdout. |
33 | quiet: Do not print messages to stdout. | |
34 | ratelimit: Download speed limit, in bytes/sec. | |
35 | retries: Number of times to retry for HTTP error 5xx | |
36 | buffersize: Size of download buffer in bytes. | |
37 | noresizebuffer: Do not automatically resize the download buffer. | |
38 | continuedl: Try to continue downloads if possible. | |
39 | noprogress: Do not print the progress bar. | |
40 | logtostderr: Log messages to stderr instead of stdout. | |
41 | consoletitle: Display progress in console window's titlebar. | |
42 | nopart: Do not use temporary .part files. | |
43 | updatetime: Use the Last-modified header to set output file timestamps. | |
44 | test: Download only first bytes to test the downloader. | |
45 | min_filesize: Skip files smaller than this size | |
46 | max_filesize: Skip files larger than this size | |
47 | xattr_set_filesize: Set ytdl.filesize user xattribute with expected size. | |
17cc1534 | 48 | (experimental) |
c75f0b36 PH |
49 | external_downloader_args: A list of additional command-line arguments for the |
50 | external downloader. | |
7d106a65 | 51 | hls_use_mpegts: Use the mpegts container for HLS videos. |
3bc2ddcc JMF |
52 | |
53 | Subclasses of this one must re-define the real_download method. | |
54 | """ | |
55 | ||
b686fc18 | 56 | _TEST_FILE_SIZE = 10241 |
3bc2ddcc JMF |
57 | params = None |
58 | ||
59 | def __init__(self, ydl, params): | |
60 | """Create a FileDownloader object with the given options.""" | |
61 | self.ydl = ydl | |
62 | self._progress_hooks = [] | |
63 | self.params = params | |
5cda4eda | 64 | self.add_progress_hook(self.report_progress) |
3bc2ddcc JMF |
65 | |
66 | @staticmethod | |
67 | def format_seconds(seconds): | |
68 | (mins, secs) = divmod(seconds, 60) | |
69 | (hours, mins) = divmod(mins, 60) | |
70 | if hours > 99: | |
71 | return '--:--:--' | |
72 | if hours == 0: | |
73 | return '%02d:%02d' % (mins, secs) | |
74 | else: | |
75 | return '%02d:%02d:%02d' % (hours, mins, secs) | |
76 | ||
77 | @staticmethod | |
78 | def calc_percent(byte_counter, data_len): | |
79 | if data_len is None: | |
80 | return None | |
81 | return float(byte_counter) / float(data_len) * 100.0 | |
82 | ||
83 | @staticmethod | |
84 | def format_percent(percent): | |
85 | if percent is None: | |
86 | return '---.-%' | |
87 | return '%6s' % ('%3.1f%%' % percent) | |
88 | ||
89 | @staticmethod | |
90 | def calc_eta(start, now, total, current): | |
91 | if total is None: | |
92 | return None | |
c7667c2d S |
93 | if now is None: |
94 | now = time.time() | |
3bc2ddcc | 95 | dif = now - start |
5f6a1245 | 96 | if current == 0 or dif < 0.001: # One millisecond |
3bc2ddcc JMF |
97 | return None |
98 | rate = float(current) / dif | |
99 | return int((float(total) - float(current)) / rate) | |
100 | ||
101 | @staticmethod | |
102 | def format_eta(eta): | |
103 | if eta is None: | |
104 | return '--:--' | |
105 | return FileDownloader.format_seconds(eta) | |
106 | ||
107 | @staticmethod | |
108 | def calc_speed(start, now, bytes): | |
109 | dif = now - start | |
5f6a1245 | 110 | if bytes == 0 or dif < 0.001: # One millisecond |
3bc2ddcc JMF |
111 | return None |
112 | return float(bytes) / dif | |
113 | ||
114 | @staticmethod | |
115 | def format_speed(speed): | |
116 | if speed is None: | |
117 | return '%10s' % '---b/s' | |
118 | return '%10s' % ('%s/s' % format_bytes(speed)) | |
119 | ||
617e58d8 S |
120 | @staticmethod |
121 | def format_retries(retries): | |
122 | return 'inf' if retries == float('inf') else '%.0f' % retries | |
123 | ||
3bc2ddcc JMF |
124 | @staticmethod |
125 | def best_block_size(elapsed_time, bytes): | |
126 | new_min = max(bytes / 2.0, 1.0) | |
5f6a1245 | 127 | new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB |
3bc2ddcc JMF |
128 | if elapsed_time < 0.001: |
129 | return int(new_max) | |
130 | rate = bytes / elapsed_time | |
131 | if rate > new_max: | |
132 | return int(new_max) | |
133 | if rate < new_min: | |
134 | return int(new_min) | |
135 | return int(rate) | |
136 | ||
137 | @staticmethod | |
138 | def parse_bytes(bytestr): | |
139 | """Parse a string indicating a byte quantity into an integer.""" | |
140 | matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) | |
141 | if matchobj is None: | |
142 | return None | |
143 | number = float(matchobj.group(1)) | |
144 | multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) | |
145 | return int(round(number * multiplier)) | |
146 | ||
147 | def to_screen(self, *args, **kargs): | |
148 | self.ydl.to_screen(*args, **kargs) | |
149 | ||
150 | def to_stderr(self, message): | |
151 | self.ydl.to_screen(message) | |
152 | ||
153 | def to_console_title(self, message): | |
154 | self.ydl.to_console_title(message) | |
155 | ||
156 | def trouble(self, *args, **kargs): | |
157 | self.ydl.trouble(*args, **kargs) | |
158 | ||
159 | def report_warning(self, *args, **kargs): | |
160 | self.ydl.report_warning(*args, **kargs) | |
161 | ||
162 | def report_error(self, *args, **kargs): | |
163 | self.ydl.report_error(*args, **kargs) | |
164 | ||
c7667c2d | 165 | def slow_down(self, start_time, now, byte_counter): |
3bc2ddcc | 166 | """Sleep if the download speed is over the rate limit.""" |
d800609c | 167 | rate_limit = self.params.get('ratelimit') |
3bc2ddcc JMF |
168 | if rate_limit is None or byte_counter == 0: |
169 | return | |
c7667c2d S |
170 | if now is None: |
171 | now = time.time() | |
3bc2ddcc JMF |
172 | elapsed = now - start_time |
173 | if elapsed <= 0.0: | |
174 | return | |
175 | speed = float(byte_counter) / elapsed | |
176 | if speed > rate_limit: | |
cc8c9281 | 177 | time.sleep(max((byte_counter // rate_limit) - elapsed, 0)) |
3bc2ddcc JMF |
178 | |
179 | def temp_name(self, filename): | |
180 | """Returns a temporary filename for the given filename.""" | |
b6b70730 | 181 | if self.params.get('nopart', False) or filename == '-' or \ |
3bc2ddcc JMF |
182 | (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): |
183 | return filename | |
b6b70730 | 184 | return filename + '.part' |
3bc2ddcc JMF |
185 | |
186 | def undo_temp_name(self, filename): | |
b6b70730 PH |
187 | if filename.endswith('.part'): |
188 | return filename[:-len('.part')] | |
3bc2ddcc JMF |
189 | return filename |
190 | ||
ea0c2f21 RA |
191 | def ytdl_filename(self, filename): |
192 | return filename + '.ytdl' | |
193 | ||
3bc2ddcc JMF |
194 | def try_rename(self, old_filename, new_filename): |
195 | try: | |
196 | if old_filename == new_filename: | |
197 | return | |
198 | os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) | |
199 | except (IOError, OSError) as err: | |
9b9c5355 | 200 | self.report_error('unable to rename file: %s' % error_to_compat_str(err)) |
3bc2ddcc JMF |
201 | |
202 | def try_utime(self, filename, last_modified_hdr): | |
203 | """Try to set the last-modified time of the given file.""" | |
204 | if last_modified_hdr is None: | |
205 | return | |
206 | if not os.path.isfile(encodeFilename(filename)): | |
207 | return | |
208 | timestr = last_modified_hdr | |
209 | if timestr is None: | |
210 | return | |
211 | filetime = timeconvert(timestr) | |
212 | if filetime is None: | |
213 | return filetime | |
214 | # Ignore obviously invalid dates | |
215 | if filetime == 0: | |
216 | return | |
217 | try: | |
218 | os.utime(filename, (time.time(), filetime)) | |
70a1165b | 219 | except Exception: |
3bc2ddcc JMF |
220 | pass |
221 | return filetime | |
222 | ||
223 | def report_destination(self, filename): | |
224 | """Report destination filename.""" | |
b6b70730 | 225 | self.to_screen('[download] Destination: ' + filename) |
3bc2ddcc JMF |
226 | |
227 | def _report_progress_status(self, msg, is_last_line=False): | |
b6b70730 | 228 | fullmsg = '[download] ' + msg |
3bc2ddcc JMF |
229 | if self.params.get('progress_with_newline', False): |
230 | self.to_screen(fullmsg) | |
231 | else: | |
e9c0cdd3 | 232 | if compat_os_name == 'nt': |
3bc2ddcc JMF |
233 | prev_len = getattr(self, '_report_progress_prev_line_length', |
234 | 0) | |
235 | if prev_len > len(fullmsg): | |
b6b70730 | 236 | fullmsg += ' ' * (prev_len - len(fullmsg)) |
3bc2ddcc | 237 | self._report_progress_prev_line_length = len(fullmsg) |
b6b70730 | 238 | clear_line = '\r' |
3bc2ddcc | 239 | else: |
b6b70730 | 240 | clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r') |
3bc2ddcc | 241 | self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) |
b6b70730 | 242 | self.to_console_title('youtube-dl ' + msg) |
3bc2ddcc | 243 | |
5cda4eda PH |
244 | def report_progress(self, s): |
245 | if s['status'] == 'finished': | |
246 | if self.params.get('noprogress', False): | |
247 | self.to_screen('[download] Download completed') | |
248 | else: | |
249 | s['_total_bytes_str'] = format_bytes(s['total_bytes']) | |
250 | if s.get('elapsed') is not None: | |
251 | s['_elapsed_str'] = self.format_seconds(s['elapsed']) | |
252 | msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s' | |
253 | else: | |
254 | msg_template = '100%% of %(_total_bytes_str)s' | |
255 | self._report_progress_status( | |
256 | msg_template % s, is_last_line=True) | |
257 | ||
258 | if self.params.get('noprogress'): | |
3bc2ddcc | 259 | return |
5cda4eda PH |
260 | |
261 | if s['status'] != 'downloading': | |
262 | return | |
263 | ||
264 | if s.get('eta') is not None: | |
265 | s['_eta_str'] = self.format_eta(s['eta']) | |
3bc2ddcc | 266 | else: |
5cda4eda | 267 | s['_eta_str'] = 'Unknown ETA' |
3bc2ddcc | 268 | |
5cda4eda PH |
269 | if s.get('total_bytes') and s.get('downloaded_bytes') is not None: |
270 | s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes']) | |
271 | elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None: | |
272 | s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate']) | |
273 | else: | |
274 | if s.get('downloaded_bytes') == 0: | |
275 | s['_percent_str'] = self.format_percent(0) | |
276 | else: | |
277 | s['_percent_str'] = 'Unknown %' | |
3bc2ddcc | 278 | |
5cda4eda PH |
279 | if s.get('speed') is not None: |
280 | s['_speed_str'] = self.format_speed(s['speed']) | |
281 | else: | |
282 | s['_speed_str'] = 'Unknown speed' | |
283 | ||
284 | if s.get('total_bytes') is not None: | |
285 | s['_total_bytes_str'] = format_bytes(s['total_bytes']) | |
286 | msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s' | |
287 | elif s.get('total_bytes_estimate') is not None: | |
288 | s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate']) | |
289 | msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s' | |
3bc2ddcc | 290 | else: |
5cda4eda PH |
291 | if s.get('downloaded_bytes') is not None: |
292 | s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes']) | |
293 | if s.get('elapsed'): | |
294 | s['_elapsed_str'] = self.format_seconds(s['elapsed']) | |
295 | msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)' | |
296 | else: | |
297 | msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s' | |
298 | else: | |
299 | msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s' | |
300 | ||
301 | self._report_progress_status(msg_template % s) | |
3bc2ddcc JMF |
302 | |
303 | def report_resuming_byte(self, resume_len): | |
304 | """Report attempt to resume at given byte.""" | |
b6b70730 | 305 | self.to_screen('[download] Resuming download at byte %s' % resume_len) |
3bc2ddcc | 306 | |
a3c3a1e1 | 307 | def report_retry(self, err, count, retries): |
3bc2ddcc | 308 | """Report retry in case of HTTP error 5xx""" |
617e58d8 | 309 | self.to_screen( |
a3c3a1e1 S |
310 | '[download] Got server HTTP error: %s. Retrying (attempt %d of %s)...' |
311 | % (error_to_compat_str(err), count, self.format_retries(retries))) | |
3bc2ddcc JMF |
312 | |
313 | def report_file_already_downloaded(self, file_name): | |
314 | """Report file has already been fully downloaded.""" | |
315 | try: | |
b6b70730 | 316 | self.to_screen('[download] %s has already been downloaded' % file_name) |
3bc2ddcc | 317 | except UnicodeEncodeError: |
b6b70730 | 318 | self.to_screen('[download] The file has already been downloaded') |
3bc2ddcc JMF |
319 | |
320 | def report_unable_to_resume(self): | |
321 | """Report it was impossible to resume download.""" | |
b6b70730 | 322 | self.to_screen('[download] Unable to resume') |
3bc2ddcc JMF |
323 | |
324 | def download(self, filename, info_dict): | |
325 | """Download to a filename using the info from info_dict | |
326 | Return True on success and False otherwise | |
327 | """ | |
5f0d813d | 328 | |
4340deca | 329 | nooverwrites_and_exists = ( |
8fb3ac36 PH |
330 | self.params.get('nooverwrites', False) and |
331 | os.path.exists(encodeFilename(filename)) | |
4340deca P |
332 | ) |
333 | ||
75a24854 RA |
334 | if not hasattr(filename, 'write'): |
335 | continuedl_and_exists = ( | |
336 | self.params.get('continuedl', True) and | |
337 | os.path.isfile(encodeFilename(filename)) and | |
338 | not self.params.get('nopart', False) | |
339 | ) | |
340 | ||
341 | # Check file already present | |
342 | if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists): | |
343 | self.report_file_already_downloaded(filename) | |
344 | self._hook_progress({ | |
345 | 'filename': filename, | |
346 | 'status': 'finished', | |
347 | 'total_bytes': os.path.getsize(encodeFilename(filename)), | |
348 | }) | |
349 | return True | |
dabc1273 | 350 | |
1ad6b891 S |
351 | min_sleep_interval = self.params.get('sleep_interval') |
352 | if min_sleep_interval: | |
353 | max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval) | |
1ad6b891 | 354 | sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval) |
d374d943 PS |
355 | self.to_screen( |
356 | '[download] Sleeping %s seconds...' % ( | |
357 | int(sleep_interval) if sleep_interval.is_integer() | |
358 | else '%.2f' % sleep_interval)) | |
5f0d813d PH |
359 | time.sleep(sleep_interval) |
360 | ||
dabc1273 | 361 | return self.real_download(filename, info_dict) |
3bc2ddcc JMF |
362 | |
363 | def real_download(self, filename, info_dict): | |
364 | """Real download process. Redefine in subclasses.""" | |
b6b70730 | 365 | raise NotImplementedError('This method must be implemented by subclasses') |
3bc2ddcc JMF |
366 | |
367 | def _hook_progress(self, status): | |
368 | for ph in self._progress_hooks: | |
369 | ph(status) | |
370 | ||
371 | def add_progress_hook(self, ph): | |
71b640cc PH |
372 | # See YoutubeDl.py (search for progress_hooks) for a description of |
373 | # this interface | |
3bc2ddcc | 374 | self._progress_hooks.append(ph) |
222516d9 | 375 | |
cd8a07a7 | 376 | def _debug_cmd(self, args, exe=None): |
222516d9 PH |
377 | if not self.params.get('verbose', False): |
378 | return | |
379 | ||
cd8a07a7 S |
380 | str_args = [decodeArgument(a) for a in args] |
381 | ||
222516d9 | 382 | if exe is None: |
cd8a07a7 | 383 | exe = os.path.basename(str_args[0]) |
222516d9 | 384 | |
222516d9 PH |
385 | self.to_screen('[debug] %s command line: %s' % ( |
386 | exe, shell_quote(str_args))) |