]>
Commit | Line | Data |
---|---|---|
b6b70730 PH |
1 | from __future__ import unicode_literals |
2 | ||
3bc2ddcc JMF |
3 | import os |
4 | import re | |
3bc2ddcc JMF |
5 | import sys |
6 | import time | |
7 | ||
1cc79574 | 8 | from ..compat import compat_str |
3bc2ddcc JMF |
9 | from ..utils import ( |
10 | encodeFilename, | |
3bc2ddcc | 11 | format_bytes, |
e3ced9ed | 12 | timeconvert, |
3bc2ddcc JMF |
13 | ) |
14 | ||
15 | ||
16 | class FileDownloader(object): | |
17 | """File Downloader class. | |
18 | ||
19 | File downloader objects are the ones responsible of downloading the | |
20 | actual video file and writing it to disk. | |
21 | ||
22 | File downloaders accept a lot of parameters. In order not to saturate | |
23 | the object constructor with arguments, it receives a dictionary of | |
24 | options instead. | |
25 | ||
26 | Available options: | |
27 | ||
881e6a1f PH |
28 | verbose: Print additional info to stdout. |
29 | quiet: Do not print messages to stdout. | |
30 | ratelimit: Download speed limit, in bytes/sec. | |
31 | retries: Number of times to retry for HTTP error 5xx | |
32 | buffersize: Size of download buffer in bytes. | |
33 | noresizebuffer: Do not automatically resize the download buffer. | |
34 | continuedl: Try to continue downloads if possible. | |
35 | noprogress: Do not print the progress bar. | |
36 | logtostderr: Log messages to stderr instead of stdout. | |
37 | consoletitle: Display progress in console window's titlebar. | |
38 | nopart: Do not use temporary .part files. | |
39 | updatetime: Use the Last-modified header to set output file timestamps. | |
40 | test: Download only first bytes to test the downloader. | |
41 | min_filesize: Skip files smaller than this size | |
42 | max_filesize: Skip files larger than this size | |
43 | xattr_set_filesize: Set ytdl.filesize user xattribute with expected size. | |
44 | (experimenatal) | |
3bc2ddcc JMF |
45 | |
46 | Subclasses of this one must re-define the real_download method. | |
47 | """ | |
48 | ||
b686fc18 | 49 | _TEST_FILE_SIZE = 10241 |
3bc2ddcc JMF |
50 | params = None |
51 | ||
52 | def __init__(self, ydl, params): | |
53 | """Create a FileDownloader object with the given options.""" | |
54 | self.ydl = ydl | |
55 | self._progress_hooks = [] | |
56 | self.params = params | |
57 | ||
58 | @staticmethod | |
59 | def format_seconds(seconds): | |
60 | (mins, secs) = divmod(seconds, 60) | |
61 | (hours, mins) = divmod(mins, 60) | |
62 | if hours > 99: | |
63 | return '--:--:--' | |
64 | if hours == 0: | |
65 | return '%02d:%02d' % (mins, secs) | |
66 | else: | |
67 | return '%02d:%02d:%02d' % (hours, mins, secs) | |
68 | ||
69 | @staticmethod | |
70 | def calc_percent(byte_counter, data_len): | |
71 | if data_len is None: | |
72 | return None | |
73 | return float(byte_counter) / float(data_len) * 100.0 | |
74 | ||
75 | @staticmethod | |
76 | def format_percent(percent): | |
77 | if percent is None: | |
78 | return '---.-%' | |
79 | return '%6s' % ('%3.1f%%' % percent) | |
80 | ||
81 | @staticmethod | |
82 | def calc_eta(start, now, total, current): | |
83 | if total is None: | |
84 | return None | |
c7667c2d S |
85 | if now is None: |
86 | now = time.time() | |
3bc2ddcc | 87 | dif = now - start |
5f6a1245 | 88 | if current == 0 or dif < 0.001: # One millisecond |
3bc2ddcc JMF |
89 | return None |
90 | rate = float(current) / dif | |
91 | return int((float(total) - float(current)) / rate) | |
92 | ||
93 | @staticmethod | |
94 | def format_eta(eta): | |
95 | if eta is None: | |
96 | return '--:--' | |
97 | return FileDownloader.format_seconds(eta) | |
98 | ||
99 | @staticmethod | |
100 | def calc_speed(start, now, bytes): | |
101 | dif = now - start | |
5f6a1245 | 102 | if bytes == 0 or dif < 0.001: # One millisecond |
3bc2ddcc JMF |
103 | return None |
104 | return float(bytes) / dif | |
105 | ||
106 | @staticmethod | |
107 | def format_speed(speed): | |
108 | if speed is None: | |
109 | return '%10s' % '---b/s' | |
110 | return '%10s' % ('%s/s' % format_bytes(speed)) | |
111 | ||
112 | @staticmethod | |
113 | def best_block_size(elapsed_time, bytes): | |
114 | new_min = max(bytes / 2.0, 1.0) | |
5f6a1245 | 115 | new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB |
3bc2ddcc JMF |
116 | if elapsed_time < 0.001: |
117 | return int(new_max) | |
118 | rate = bytes / elapsed_time | |
119 | if rate > new_max: | |
120 | return int(new_max) | |
121 | if rate < new_min: | |
122 | return int(new_min) | |
123 | return int(rate) | |
124 | ||
125 | @staticmethod | |
126 | def parse_bytes(bytestr): | |
127 | """Parse a string indicating a byte quantity into an integer.""" | |
128 | matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) | |
129 | if matchobj is None: | |
130 | return None | |
131 | number = float(matchobj.group(1)) | |
132 | multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) | |
133 | return int(round(number * multiplier)) | |
134 | ||
135 | def to_screen(self, *args, **kargs): | |
136 | self.ydl.to_screen(*args, **kargs) | |
137 | ||
138 | def to_stderr(self, message): | |
139 | self.ydl.to_screen(message) | |
140 | ||
141 | def to_console_title(self, message): | |
142 | self.ydl.to_console_title(message) | |
143 | ||
144 | def trouble(self, *args, **kargs): | |
145 | self.ydl.trouble(*args, **kargs) | |
146 | ||
147 | def report_warning(self, *args, **kargs): | |
148 | self.ydl.report_warning(*args, **kargs) | |
149 | ||
150 | def report_error(self, *args, **kargs): | |
151 | self.ydl.report_error(*args, **kargs) | |
152 | ||
c7667c2d | 153 | def slow_down(self, start_time, now, byte_counter): |
3bc2ddcc JMF |
154 | """Sleep if the download speed is over the rate limit.""" |
155 | rate_limit = self.params.get('ratelimit', None) | |
156 | if rate_limit is None or byte_counter == 0: | |
157 | return | |
c7667c2d S |
158 | if now is None: |
159 | now = time.time() | |
3bc2ddcc JMF |
160 | elapsed = now - start_time |
161 | if elapsed <= 0.0: | |
162 | return | |
163 | speed = float(byte_counter) / elapsed | |
164 | if speed > rate_limit: | |
cc8c9281 | 165 | time.sleep(max((byte_counter // rate_limit) - elapsed, 0)) |
3bc2ddcc JMF |
166 | |
167 | def temp_name(self, filename): | |
168 | """Returns a temporary filename for the given filename.""" | |
b6b70730 | 169 | if self.params.get('nopart', False) or filename == '-' or \ |
3bc2ddcc JMF |
170 | (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): |
171 | return filename | |
b6b70730 | 172 | return filename + '.part' |
3bc2ddcc JMF |
173 | |
174 | def undo_temp_name(self, filename): | |
b6b70730 PH |
175 | if filename.endswith('.part'): |
176 | return filename[:-len('.part')] | |
3bc2ddcc JMF |
177 | return filename |
178 | ||
179 | def try_rename(self, old_filename, new_filename): | |
180 | try: | |
181 | if old_filename == new_filename: | |
182 | return | |
183 | os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) | |
184 | except (IOError, OSError) as err: | |
b6b70730 | 185 | self.report_error('unable to rename file: %s' % compat_str(err)) |
3bc2ddcc JMF |
186 | |
187 | def try_utime(self, filename, last_modified_hdr): | |
188 | """Try to set the last-modified time of the given file.""" | |
189 | if last_modified_hdr is None: | |
190 | return | |
191 | if not os.path.isfile(encodeFilename(filename)): | |
192 | return | |
193 | timestr = last_modified_hdr | |
194 | if timestr is None: | |
195 | return | |
196 | filetime = timeconvert(timestr) | |
197 | if filetime is None: | |
198 | return filetime | |
199 | # Ignore obviously invalid dates | |
200 | if filetime == 0: | |
201 | return | |
202 | try: | |
203 | os.utime(filename, (time.time(), filetime)) | |
204 | except: | |
205 | pass | |
206 | return filetime | |
207 | ||
208 | def report_destination(self, filename): | |
209 | """Report destination filename.""" | |
b6b70730 | 210 | self.to_screen('[download] Destination: ' + filename) |
3bc2ddcc JMF |
211 | |
212 | def _report_progress_status(self, msg, is_last_line=False): | |
b6b70730 | 213 | fullmsg = '[download] ' + msg |
3bc2ddcc JMF |
214 | if self.params.get('progress_with_newline', False): |
215 | self.to_screen(fullmsg) | |
216 | else: | |
217 | if os.name == 'nt': | |
218 | prev_len = getattr(self, '_report_progress_prev_line_length', | |
219 | 0) | |
220 | if prev_len > len(fullmsg): | |
b6b70730 | 221 | fullmsg += ' ' * (prev_len - len(fullmsg)) |
3bc2ddcc | 222 | self._report_progress_prev_line_length = len(fullmsg) |
b6b70730 | 223 | clear_line = '\r' |
3bc2ddcc | 224 | else: |
b6b70730 | 225 | clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r') |
3bc2ddcc | 226 | self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) |
b6b70730 | 227 | self.to_console_title('youtube-dl ' + msg) |
3bc2ddcc JMF |
228 | |
229 | def report_progress(self, percent, data_len_str, speed, eta): | |
230 | """Report download progress.""" | |
231 | if self.params.get('noprogress', False): | |
232 | return | |
233 | if eta is not None: | |
234 | eta_str = self.format_eta(eta) | |
235 | else: | |
236 | eta_str = 'Unknown ETA' | |
237 | if percent is not None: | |
238 | percent_str = self.format_percent(percent) | |
239 | else: | |
240 | percent_str = 'Unknown %' | |
241 | speed_str = self.format_speed(speed) | |
242 | ||
b6b70730 | 243 | msg = ('%s of %s at %s ETA %s' % |
3bc2ddcc JMF |
244 | (percent_str, data_len_str, speed_str, eta_str)) |
245 | self._report_progress_status(msg) | |
246 | ||
247 | def report_progress_live_stream(self, downloaded_data_len, speed, elapsed): | |
248 | if self.params.get('noprogress', False): | |
249 | return | |
250 | downloaded_str = format_bytes(downloaded_data_len) | |
251 | speed_str = self.format_speed(speed) | |
252 | elapsed_str = FileDownloader.format_seconds(elapsed) | |
b6b70730 | 253 | msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str) |
3bc2ddcc JMF |
254 | self._report_progress_status(msg) |
255 | ||
256 | def report_finish(self, data_len_str, tot_time): | |
257 | """Report download finished.""" | |
258 | if self.params.get('noprogress', False): | |
b6b70730 | 259 | self.to_screen('[download] Download completed') |
3bc2ddcc JMF |
260 | else: |
261 | self._report_progress_status( | |
b6b70730 | 262 | ('100%% of %s in %s' % |
3bc2ddcc JMF |
263 | (data_len_str, self.format_seconds(tot_time))), |
264 | is_last_line=True) | |
265 | ||
266 | def report_resuming_byte(self, resume_len): | |
267 | """Report attempt to resume at given byte.""" | |
b6b70730 | 268 | self.to_screen('[download] Resuming download at byte %s' % resume_len) |
3bc2ddcc JMF |
269 | |
270 | def report_retry(self, count, retries): | |
271 | """Report retry in case of HTTP error 5xx""" | |
b6b70730 | 272 | self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) |
3bc2ddcc JMF |
273 | |
274 | def report_file_already_downloaded(self, file_name): | |
275 | """Report file has already been fully downloaded.""" | |
276 | try: | |
b6b70730 | 277 | self.to_screen('[download] %s has already been downloaded' % file_name) |
3bc2ddcc | 278 | except UnicodeEncodeError: |
b6b70730 | 279 | self.to_screen('[download] The file has already been downloaded') |
3bc2ddcc JMF |
280 | |
281 | def report_unable_to_resume(self): | |
282 | """Report it was impossible to resume download.""" | |
b6b70730 | 283 | self.to_screen('[download] Unable to resume') |
3bc2ddcc JMF |
284 | |
285 | def download(self, filename, info_dict): | |
286 | """Download to a filename using the info from info_dict | |
287 | Return True on success and False otherwise | |
288 | """ | |
5f0d813d | 289 | |
4340deca P |
290 | nooverwrites_and_exists = ( |
291 | self.params.get('nooverwrites', False) | |
292 | and os.path.exists(encodeFilename(filename)) | |
293 | ) | |
294 | ||
295 | continuedl_and_exists = ( | |
296 | self.params.get('continuedl', False) | |
297 | and os.path.isfile(encodeFilename(filename)) | |
298 | and not self.params.get('nopart', False) | |
299 | ) | |
300 | ||
3bc2ddcc | 301 | # Check file already present |
4340deca | 302 | if filename != '-' and nooverwrites_and_exists or continuedl_and_exists: |
3bc2ddcc JMF |
303 | self.report_file_already_downloaded(filename) |
304 | self._hook_progress({ | |
305 | 'filename': filename, | |
306 | 'status': 'finished', | |
307 | 'total_bytes': os.path.getsize(encodeFilename(filename)), | |
308 | }) | |
309 | return True | |
dabc1273 | 310 | |
5f0d813d PH |
311 | sleep_interval = self.params.get('sleep_interval') |
312 | if sleep_interval: | |
313 | self.to_screen('[download] Sleeping %s seconds...' % sleep_interval) | |
314 | time.sleep(sleep_interval) | |
315 | ||
dabc1273 | 316 | return self.real_download(filename, info_dict) |
3bc2ddcc JMF |
317 | |
318 | def real_download(self, filename, info_dict): | |
319 | """Real download process. Redefine in subclasses.""" | |
b6b70730 | 320 | raise NotImplementedError('This method must be implemented by subclasses') |
3bc2ddcc JMF |
321 | |
322 | def _hook_progress(self, status): | |
323 | for ph in self._progress_hooks: | |
324 | ph(status) | |
325 | ||
326 | def add_progress_hook(self, ph): | |
71b640cc PH |
327 | # See YoutubeDl.py (search for progress_hooks) for a description of |
328 | # this interface | |
3bc2ddcc | 329 | self._progress_hooks.append(ph) |
222516d9 PH |
330 | |
331 | def _debug_cmd(self, args, subprocess_encoding, exe=None): | |
332 | if not self.params.get('verbose', False): | |
333 | return | |
334 | ||
335 | if exe is None: | |
336 | exe = os.path.basename(args[0]) | |
337 | ||
338 | if subprocess_encoding: | |
339 | str_args = [ | |
340 | a.decode(subprocess_encoding) if isinstance(a, bytes) else a | |
341 | for a in args] | |
342 | else: | |
343 | str_args = args | |
344 | try: | |
345 | import pipes | |
346 | shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) | |
347 | except ImportError: | |
348 | shell_quote = repr | |
349 | self.to_screen('[debug] %s command line: %s' % ( | |
350 | exe, shell_quote(str_args))) |