yt_dlp/downloader/common.py

   1 from __future__ import division, unicode_literals
   2
   3 import copy
   4 import os
   5 import re
   6 import sys
   7 import time
   8 import random
   9
  10 from ..compat import compat_os_name
  11 from ..utils import (
  12     decodeArgument,
  13     encodeFilename,
  14     error_to_compat_str,
  15     format_bytes,
  16     shell_quote,
  17     timeconvert,
  18 )
  19
  20
  21 class FileDownloader(object):
  22     """File Downloader class.
  23
  24     File downloader objects are the ones responsible of downloading the
  25     actual video file and writing it to disk.
  26
  27     File downloaders accept a lot of parameters. In order not to saturate
  28     the object constructor with arguments, it receives a dictionary of
  29     options instead.
  30
  31     Available options:
  32
  33     verbose:            Print additional info to stdout.
  34     quiet:              Do not print messages to stdout.
  35     ratelimit:          Download speed limit, in bytes/sec.
  36     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
  37     retries:            Number of times to retry for HTTP error 5xx
  38     buffersize:         Size of download buffer in bytes.
  39     noresizebuffer:     Do not automatically resize the download buffer.
  40     continuedl:         Try to continue downloads if possible.
  41     noprogress:         Do not print the progress bar.
  42     logtostderr:        Log messages to stderr instead of stdout.
  43     consoletitle:       Display progress in console window's titlebar.
  44     nopart:             Do not use temporary .part files.
  45     updatetime:         Use the Last-modified header to set output file timestamps.
  46     test:               Download only first bytes to test the downloader.
  47     min_filesize:       Skip files smaller than this size
  48     max_filesize:       Skip files larger than this size
  49     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
  50     external_downloader_args:  A dictionary of downloader keys (in lower case)
  51                         and a list of additional command-line arguments for the
  52                         executable. Use 'default' as the name for arguments to be
  53                         passed to all downloaders. For compatibility with youtube-dl,
  54                         a single list of args can also be used
  55     hls_use_mpegts:     Use the mpegts container for HLS videos.
  56     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
  57                         useful for bypassing bandwidth throttling imposed by
  58                         a webserver (experimental)
  59
  60     Subclasses of this one must re-define the real_download method.
  61     """
  62
  63     _TEST_FILE_SIZE = 10241
  64     params = None
  65
  66     def __init__(self, ydl, params):
  67         """Create a FileDownloader object with the given options."""
  68         self.ydl = ydl
  69         self._progress_hooks = []
  70         self.params = params
  71         self.add_progress_hook(self.report_progress)
  72
  73     @staticmethod
  74     def format_seconds(seconds):
  75         (mins, secs) = divmod(seconds, 60)
  76         (hours, mins) = divmod(mins, 60)
  77         if hours > 99:
  78             return '--:--:--'
  79         if hours == 0:
  80             return '%02d:%02d' % (mins, secs)
  81         else:
  82             return '%02d:%02d:%02d' % (hours, mins, secs)
  83
  84     @staticmethod
  85     def calc_percent(byte_counter, data_len):
  86         if data_len is None:
  87             return None
  88         return float(byte_counter) / float(data_len) * 100.0
  89
  90     @staticmethod
  91     def format_percent(percent):
  92         if percent is None:
  93             return '---.-%'
  94         return '%6s' % ('%3.1f%%' % percent)
  95
  96     @staticmethod
  97     def calc_eta(start, now, total, current):
  98         if total is None:
  99             return None
 100         if now is None:
 101             now = time.time()
 102         dif = now - start
 103         if current == 0 or dif < 0.001:  # One millisecond
 104             return None
 105         rate = float(current) / dif
 106         return int((float(total) - float(current)) / rate)
 107
 108     @staticmethod
 109     def format_eta(eta):
 110         if eta is None:
 111             return '--:--'
 112         return FileDownloader.format_seconds(eta)
 113
 114     @staticmethod
 115     def calc_speed(start, now, bytes):
 116         dif = now - start
 117         if bytes == 0 or dif < 0.001:  # One millisecond
 118             return None
 119         return float(bytes) / dif
 120
 121     @staticmethod
 122     def format_speed(speed):
 123         if speed is None:
 124             return '%10s' % '---b/s'
 125         return '%10s' % ('%s/s' % format_bytes(speed))
 126
 127     @staticmethod
 128     def format_retries(retries):
 129         return 'inf' if retries == float('inf') else '%.0f' % retries
 130
 131     @staticmethod
 132     def best_block_size(elapsed_time, bytes):
 133         new_min = max(bytes / 2.0, 1.0)
 134         new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
 135         if elapsed_time < 0.001:
 136             return int(new_max)
 137         rate = bytes / elapsed_time
 138         if rate > new_max:
 139             return int(new_max)
 140         if rate < new_min:
 141             return int(new_min)
 142         return int(rate)
 143
 144     @staticmethod
 145     def parse_bytes(bytestr):
 146         """Parse a string indicating a byte quantity into an integer."""
 147         matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 148         if matchobj is None:
 149             return None
 150         number = float(matchobj.group(1))
 151         multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 152         return int(round(number * multiplier))
 153
 154     def to_screen(self, *args, **kargs):
 155         self.ydl.to_stdout(*args, quiet=self.params.get('quiet'), **kargs)
 156
 157     def to_stderr(self, message):
 158         self.ydl.to_stderr(message)
 159
 160     def to_console_title(self, message):
 161         self.ydl.to_console_title(message)
 162
 163     def trouble(self, *args, **kargs):
 164         self.ydl.trouble(*args, **kargs)
 165
 166     def report_warning(self, *args, **kargs):
 167         self.ydl.report_warning(*args, **kargs)
 168
 169     def report_error(self, *args, **kargs):
 170         self.ydl.report_error(*args, **kargs)
 171
 172     def write_debug(self, *args, **kargs):
 173         self.ydl.write_debug(*args, **kargs)
 174
 175     def slow_down(self, start_time, now, byte_counter):
 176         """Sleep if the download speed is over the rate limit."""
 177         rate_limit = self.params.get('ratelimit')
 178         if rate_limit is None or byte_counter == 0:
 179             return
 180         if now is None:
 181             now = time.time()
 182         elapsed = now - start_time
 183         if elapsed <= 0.0:
 184             return
 185         speed = float(byte_counter) / elapsed
 186         if speed > rate_limit:
 187             sleep_time = float(byte_counter) / rate_limit - elapsed
 188             if sleep_time > 0:
 189                 time.sleep(sleep_time)
 190
 191     def temp_name(self, filename):
 192         """Returns a temporary filename for the given filename."""
 193         if self.params.get('nopart', False) or filename == '-' or \
 194                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 195             return filename
 196         return filename + '.part'
 197
 198     def undo_temp_name(self, filename):
 199         if filename.endswith('.part'):
 200             return filename[:-len('.part')]
 201         return filename
 202
 203     def ytdl_filename(self, filename):
 204         return filename + '.ytdl'
 205
 206     def try_rename(self, old_filename, new_filename):
 207         if old_filename == new_filename:
 208             return
 209         try:
 210             if self.params.get('overwrites', False):
 211                 if os.path.isfile(encodeFilename(new_filename)):
 212                     os.remove(encodeFilename(new_filename))
 213             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 214         except (IOError, OSError) as err:
 215             self.report_error('unable to rename file: %s' % error_to_compat_str(err))
 216
 217     def try_utime(self, filename, last_modified_hdr):
 218         """Try to set the last-modified time of the given file."""
 219         if last_modified_hdr is None:
 220             return
 221         if not os.path.isfile(encodeFilename(filename)):
 222             return
 223         timestr = last_modified_hdr
 224         if timestr is None:
 225             return
 226         filetime = timeconvert(timestr)
 227         if filetime is None:
 228             return filetime
 229         # Ignore obviously invalid dates
 230         if filetime == 0:
 231             return
 232         try:
 233             os.utime(filename, (time.time(), filetime))
 234         except Exception:
 235             pass
 236         return filetime
 237
 238     def report_destination(self, filename):
 239         """Report destination filename."""
 240         self.to_screen('[download] Destination: ' + filename)
 241
 242     def _report_progress_status(self, msg, is_last_line=False):
 243         fullmsg = '[download] ' + msg
 244         if self.params.get('progress_with_newline', False):
 245             self.to_screen(fullmsg)
 246         else:
 247             if compat_os_name == 'nt':
 248                 prev_len = getattr(self, '_report_progress_prev_line_length',
 249                                    0)
 250                 if prev_len > len(fullmsg):
 251                     fullmsg += ' ' * (prev_len - len(fullmsg))
 252                 self._report_progress_prev_line_length = len(fullmsg)
 253                 clear_line = '\r'
 254             else:
 255                 clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
 256             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
 257         self.to_console_title('yt-dlp ' + msg)
 258
 259     def report_progress(self, s):
 260         if s['status'] == 'finished':
 261             if self.params.get('noprogress', False):
 262                 self.to_screen('[download] Download completed')
 263             else:
 264                 msg_template = '100%%'
 265                 if s.get('total_bytes') is not None:
 266                     s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 267                     msg_template += ' of %(_total_bytes_str)s'
 268                 if s.get('elapsed') is not None:
 269                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 270                     msg_template += ' in %(_elapsed_str)s'
 271                 self._report_progress_status(
 272                     msg_template % s, is_last_line=True)
 273
 274         if self.params.get('noprogress'):
 275             return
 276
 277         if s['status'] != 'downloading':
 278             return
 279
 280         if s.get('eta') is not None:
 281             s['_eta_str'] = self.format_eta(s['eta'])
 282         else:
 283             s['_eta_str'] = 'Unknown ETA'
 284
 285         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
 286             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
 287         elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
 288             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
 289         else:
 290             if s.get('downloaded_bytes') == 0:
 291                 s['_percent_str'] = self.format_percent(0)
 292             else:
 293                 s['_percent_str'] = 'Unknown %'
 294
 295         if s.get('speed') is not None:
 296             s['_speed_str'] = self.format_speed(s['speed'])
 297         else:
 298             s['_speed_str'] = 'Unknown speed'
 299
 300         if s.get('total_bytes') is not None:
 301             s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 302             msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
 303         elif s.get('total_bytes_estimate') is not None:
 304             s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
 305             msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
 306         else:
 307             if s.get('downloaded_bytes') is not None:
 308                 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
 309                 if s.get('elapsed'):
 310                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 311                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
 312                 else:
 313                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
 314             else:
 315                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
 316
 317         self._report_progress_status(msg_template % s)
 318
 319     def report_resuming_byte(self, resume_len):
 320         """Report attempt to resume at given byte."""
 321         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 322
 323     def report_retry(self, err, count, retries):
 324         """Report retry in case of HTTP error 5xx"""
 325         self.to_screen(
 326             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
 327             % (error_to_compat_str(err), count, self.format_retries(retries)))
 328
 329     def report_file_already_downloaded(self, *args, **kwargs):
 330         """Report file has already been fully downloaded."""
 331         return self.ydl.report_file_already_downloaded(*args, **kwargs)
 332
 333     def report_unable_to_resume(self):
 334         """Report it was impossible to resume download."""
 335         self.to_screen('[download] Unable to resume')
 336
 337     @staticmethod
 338     def supports_manifest(manifest):
 339         """ Whether the downloader can download the fragments from the manifest.
 340         Redefine in subclasses if needed. """
 341         pass
 342
 343     def download(self, filename, info_dict, subtitle=False):
 344         """Download to a filename using the info from info_dict
 345         Return True on success and False otherwise
 346         """
 347
 348         nooverwrites_and_exists = (
 349             not self.params.get('overwrites', True)
 350             and os.path.exists(encodeFilename(filename))
 351         )
 352
 353         if not hasattr(filename, 'write'):
 354             continuedl_and_exists = (
 355                 self.params.get('continuedl', True)
 356                 and os.path.isfile(encodeFilename(filename))
 357                 and not self.params.get('nopart', False)
 358             )
 359
 360             # Check file already present
 361             if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
 362                 self.report_file_already_downloaded(filename)
 363                 self._hook_progress({
 364                     'filename': filename,
 365                     'status': 'finished',
 366                     'total_bytes': os.path.getsize(encodeFilename(filename)),
 367                 }, info_dict)
 368                 return True, False
 369
 370         if subtitle is False:
 371             min_sleep_interval = self.params.get('sleep_interval')
 372             if min_sleep_interval:
 373                 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
 374                 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
 375                 self.to_screen(
 376                     '[download] Sleeping %s seconds ...' % (
 377                         int(sleep_interval) if sleep_interval.is_integer()
 378                         else '%.2f' % sleep_interval))
 379                 time.sleep(sleep_interval)
 380         else:
 381             sleep_interval_sub = 0
 382             if type(self.params.get('sleep_interval_subtitles')) is int:
 383                 sleep_interval_sub = self.params.get('sleep_interval_subtitles')
 384             if sleep_interval_sub > 0:
 385                 self.to_screen(
 386                     '[download] Sleeping %s seconds ...' % (
 387                         sleep_interval_sub))
 388                 time.sleep(sleep_interval_sub)
 389         return self.real_download(filename, info_dict), True
 390
 391     def real_download(self, filename, info_dict):
 392         """Real download process. Redefine in subclasses."""
 393         raise NotImplementedError('This method must be implemented by subclasses')
 394
 395     def _hook_progress(self, status, info_dict):
 396         if not self._progress_hooks:
 397             return
 398         info_dict = dict(info_dict)
 399         for key in ('__original_infodict', '__postprocessors'):
 400             info_dict.pop(key, None)
 401         # youtube-dl passes the same status object to all the hooks.
 402         # Some third party scripts seems to be relying on this.
 403         # So keep this behavior if possible
 404         status['info_dict'] = copy.deepcopy(info_dict)
 405         for ph in self._progress_hooks:
 406             ph(status)
 407
 408     def add_progress_hook(self, ph):
 409         # See YoutubeDl.py (search for progress_hooks) for a description of
 410         # this interface
 411         self._progress_hooks.append(ph)
 412
 413     def _debug_cmd(self, args, exe=None):
 414         if not self.params.get('verbose', False):
 415             return
 416
 417         str_args = [decodeArgument(a) for a in args]
 418
 419         if exe is None:
 420             exe = os.path.basename(str_args[0])
 421
 422         self.write_debug('%s command line: %s' % (exe, shell_quote(str_args)))