yt_dlp/downloader/common.py

   1 from __future__ import division, unicode_literals
   2
   3 import copy
   4 import os
   5 import re
   6 import sys
   7 import time
   8 import random
   9
  10 from ..utils import (
  11     decodeArgument,
  12     encodeFilename,
  13     error_to_compat_str,
  14     format_bytes,
  15     shell_quote,
  16     timeconvert,
  17 )
  18 from ..minicurses import (
  19     MultilineLogger,
  20     MultilinePrinter,
  21     QuietMultilinePrinter,
  22     BreaklineStatusPrinter
  23 )
  24
  25
  26 class FileDownloader(object):
  27     """File Downloader class.
  28
  29     File downloader objects are the ones responsible of downloading the
  30     actual video file and writing it to disk.
  31
  32     File downloaders accept a lot of parameters. In order not to saturate
  33     the object constructor with arguments, it receives a dictionary of
  34     options instead.
  35
  36     Available options:
  37
  38     verbose:            Print additional info to stdout.
  39     quiet:              Do not print messages to stdout.
  40     ratelimit:          Download speed limit, in bytes/sec.
  41     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
  42     retries:            Number of times to retry for HTTP error 5xx
  43     buffersize:         Size of download buffer in bytes.
  44     noresizebuffer:     Do not automatically resize the download buffer.
  45     continuedl:         Try to continue downloads if possible.
  46     noprogress:         Do not print the progress bar.
  47     nopart:             Do not use temporary .part files.
  48     updatetime:         Use the Last-modified header to set output file timestamps.
  49     test:               Download only first bytes to test the downloader.
  50     min_filesize:       Skip files smaller than this size
  51     max_filesize:       Skip files larger than this size
  52     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
  53     external_downloader_args:  A dictionary of downloader keys (in lower case)
  54                         and a list of additional command-line arguments for the
  55                         executable. Use 'default' as the name for arguments to be
  56                         passed to all downloaders. For compatibility with youtube-dl,
  57                         a single list of args can also be used
  58     hls_use_mpegts:     Use the mpegts container for HLS videos.
  59     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
  60                         useful for bypassing bandwidth throttling imposed by
  61                         a webserver (experimental)
  62     progress_template:  See YoutubeDL.py
  63
  64     Subclasses of this one must re-define the real_download method.
  65     """
  66
  67     _TEST_FILE_SIZE = 10241
  68     params = None
  69
  70     def __init__(self, ydl, params):
  71         """Create a FileDownloader object with the given options."""
  72         self.ydl = ydl
  73         self._progress_hooks = []
  74         self.params = params
  75         self._prepare_multiline_status()
  76         self.add_progress_hook(self.report_progress)
  77
  78     @staticmethod
  79     def format_seconds(seconds):
  80         (mins, secs) = divmod(seconds, 60)
  81         (hours, mins) = divmod(mins, 60)
  82         if hours > 99:
  83             return '--:--:--'
  84         if hours == 0:
  85             return '%02d:%02d' % (mins, secs)
  86         else:
  87             return '%02d:%02d:%02d' % (hours, mins, secs)
  88
  89     @staticmethod
  90     def calc_percent(byte_counter, data_len):
  91         if data_len is None:
  92             return None
  93         return float(byte_counter) / float(data_len) * 100.0
  94
  95     @staticmethod
  96     def format_percent(percent):
  97         if percent is None:
  98             return '---.-%'
  99         return '%6s' % ('%3.1f%%' % percent)
 100
 101     @staticmethod
 102     def calc_eta(start, now, total, current):
 103         if total is None:
 104             return None
 105         if now is None:
 106             now = time.time()
 107         dif = now - start
 108         if current == 0 or dif < 0.001:  # One millisecond
 109             return None
 110         rate = float(current) / dif
 111         return int((float(total) - float(current)) / rate)
 112
 113     @staticmethod
 114     def format_eta(eta):
 115         if eta is None:
 116             return '--:--'
 117         return FileDownloader.format_seconds(eta)
 118
 119     @staticmethod
 120     def calc_speed(start, now, bytes):
 121         dif = now - start
 122         if bytes == 0 or dif < 0.001:  # One millisecond
 123             return None
 124         return float(bytes) / dif
 125
 126     @staticmethod
 127     def format_speed(speed):
 128         if speed is None:
 129             return '%10s' % '---b/s'
 130         return '%10s' % ('%s/s' % format_bytes(speed))
 131
 132     @staticmethod
 133     def format_retries(retries):
 134         return 'inf' if retries == float('inf') else '%.0f' % retries
 135
 136     @staticmethod
 137     def best_block_size(elapsed_time, bytes):
 138         new_min = max(bytes / 2.0, 1.0)
 139         new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
 140         if elapsed_time < 0.001:
 141             return int(new_max)
 142         rate = bytes / elapsed_time
 143         if rate > new_max:
 144             return int(new_max)
 145         if rate < new_min:
 146             return int(new_min)
 147         return int(rate)
 148
 149     @staticmethod
 150     def parse_bytes(bytestr):
 151         """Parse a string indicating a byte quantity into an integer."""
 152         matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 153         if matchobj is None:
 154             return None
 155         number = float(matchobj.group(1))
 156         multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 157         return int(round(number * multiplier))
 158
 159     def to_screen(self, *args, **kargs):
 160         self.ydl.to_stdout(*args, quiet=self.params.get('quiet'), **kargs)
 161
 162     def to_stderr(self, message):
 163         self.ydl.to_stderr(message)
 164
 165     def to_console_title(self, message):
 166         self.ydl.to_console_title(message)
 167
 168     def trouble(self, *args, **kargs):
 169         self.ydl.trouble(*args, **kargs)
 170
 171     def report_warning(self, *args, **kargs):
 172         self.ydl.report_warning(*args, **kargs)
 173
 174     def report_error(self, *args, **kargs):
 175         self.ydl.report_error(*args, **kargs)
 176
 177     def write_debug(self, *args, **kargs):
 178         self.ydl.write_debug(*args, **kargs)
 179
 180     def slow_down(self, start_time, now, byte_counter):
 181         """Sleep if the download speed is over the rate limit."""
 182         rate_limit = self.params.get('ratelimit')
 183         if rate_limit is None or byte_counter == 0:
 184             return
 185         if now is None:
 186             now = time.time()
 187         elapsed = now - start_time
 188         if elapsed <= 0.0:
 189             return
 190         speed = float(byte_counter) / elapsed
 191         if speed > rate_limit:
 192             sleep_time = float(byte_counter) / rate_limit - elapsed
 193             if sleep_time > 0:
 194                 time.sleep(sleep_time)
 195
 196     def temp_name(self, filename):
 197         """Returns a temporary filename for the given filename."""
 198         if self.params.get('nopart', False) or filename == '-' or \
 199                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 200             return filename
 201         return filename + '.part'
 202
 203     def undo_temp_name(self, filename):
 204         if filename.endswith('.part'):
 205             return filename[:-len('.part')]
 206         return filename
 207
 208     def ytdl_filename(self, filename):
 209         return filename + '.ytdl'
 210
 211     def try_rename(self, old_filename, new_filename):
 212         if old_filename == new_filename:
 213             return
 214         try:
 215             os.replace(old_filename, new_filename)
 216         except (IOError, OSError) as err:
 217             self.report_error(f'unable to rename file: {err}')
 218
 219     def try_utime(self, filename, last_modified_hdr):
 220         """Try to set the last-modified time of the given file."""
 221         if last_modified_hdr is None:
 222             return
 223         if not os.path.isfile(encodeFilename(filename)):
 224             return
 225         timestr = last_modified_hdr
 226         if timestr is None:
 227             return
 228         filetime = timeconvert(timestr)
 229         if filetime is None:
 230             return filetime
 231         # Ignore obviously invalid dates
 232         if filetime == 0:
 233             return
 234         try:
 235             os.utime(filename, (time.time(), filetime))
 236         except Exception:
 237             pass
 238         return filetime
 239
 240     def report_destination(self, filename):
 241         """Report destination filename."""
 242         self.to_screen('[download] Destination: ' + filename)
 243
 244     def _prepare_multiline_status(self, lines=1):
 245         if self.params.get('noprogress'):
 246             self._multiline = QuietMultilinePrinter()
 247         elif self.ydl.params.get('logger'):
 248             self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
 249         elif self.params.get('progress_with_newline'):
 250             self._multiline = BreaklineStatusPrinter(sys.stderr, lines)
 251         else:
 252             self._multiline = MultilinePrinter(sys.stderr, lines, not self.params.get('quiet'))
 253
 254     def _finish_multiline_status(self):
 255         self._multiline.end()
 256
 257     def _report_progress_status(self, s):
 258         progress_dict = s.copy()
 259         progress_dict.pop('info_dict')
 260         progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
 261
 262         progress_template = self.params.get('progress_template', {})
 263         self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
 264             progress_template.get('download') or '[download] %(progress._default_template)s',
 265             progress_dict), s.get('progress_idx') or 0)
 266         self.to_console_title(self.ydl.evaluate_outtmpl(
 267             progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
 268             progress_dict))
 269
 270     def report_progress(self, s):
 271         if s['status'] == 'finished':
 272             if self.params.get('noprogress'):
 273                 self.to_screen('[download] Download completed')
 274             msg_template = '100%%'
 275             if s.get('total_bytes') is not None:
 276                 s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 277                 msg_template += ' of %(_total_bytes_str)s'
 278             if s.get('elapsed') is not None:
 279                 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 280                 msg_template += ' in %(_elapsed_str)s'
 281             s['_percent_str'] = self.format_percent(100)
 282             s['_default_template'] = msg_template % s
 283             self._report_progress_status(s)
 284             return
 285
 286         if s['status'] != 'downloading':
 287             return
 288
 289         if s.get('eta') is not None:
 290             s['_eta_str'] = self.format_eta(s['eta'])
 291         else:
 292             s['_eta_str'] = 'Unknown ETA'
 293
 294         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
 295             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
 296         elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
 297             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
 298         else:
 299             if s.get('downloaded_bytes') == 0:
 300                 s['_percent_str'] = self.format_percent(0)
 301             else:
 302                 s['_percent_str'] = 'Unknown %'
 303
 304         if s.get('speed') is not None:
 305             s['_speed_str'] = self.format_speed(s['speed'])
 306         else:
 307             s['_speed_str'] = 'Unknown speed'
 308
 309         if s.get('total_bytes') is not None:
 310             s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 311             msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
 312         elif s.get('total_bytes_estimate') is not None:
 313             s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
 314             msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
 315         else:
 316             if s.get('downloaded_bytes') is not None:
 317                 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
 318                 if s.get('elapsed'):
 319                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 320                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
 321                 else:
 322                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
 323             else:
 324                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
 325         s['_default_template'] = msg_template % s
 326         self._report_progress_status(s)
 327
 328     def report_resuming_byte(self, resume_len):
 329         """Report attempt to resume at given byte."""
 330         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 331
 332     def report_retry(self, err, count, retries):
 333         """Report retry in case of HTTP error 5xx"""
 334         self.to_screen(
 335             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
 336             % (error_to_compat_str(err), count, self.format_retries(retries)))
 337
 338     def report_file_already_downloaded(self, *args, **kwargs):
 339         """Report file has already been fully downloaded."""
 340         return self.ydl.report_file_already_downloaded(*args, **kwargs)
 341
 342     def report_unable_to_resume(self):
 343         """Report it was impossible to resume download."""
 344         self.to_screen('[download] Unable to resume')
 345
 346     @staticmethod
 347     def supports_manifest(manifest):
 348         """ Whether the downloader can download the fragments from the manifest.
 349         Redefine in subclasses if needed. """
 350         pass
 351
 352     def download(self, filename, info_dict, subtitle=False):
 353         """Download to a filename using the info from info_dict
 354         Return True on success and False otherwise
 355         """
 356
 357         nooverwrites_and_exists = (
 358             not self.params.get('overwrites', True)
 359             and os.path.exists(encodeFilename(filename))
 360         )
 361
 362         if not hasattr(filename, 'write'):
 363             continuedl_and_exists = (
 364                 self.params.get('continuedl', True)
 365                 and os.path.isfile(encodeFilename(filename))
 366                 and not self.params.get('nopart', False)
 367             )
 368
 369             # Check file already present
 370             if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
 371                 self.report_file_already_downloaded(filename)
 372                 self._hook_progress({
 373                     'filename': filename,
 374                     'status': 'finished',
 375                     'total_bytes': os.path.getsize(encodeFilename(filename)),
 376                 }, info_dict)
 377                 return True, False
 378
 379         if subtitle is False:
 380             min_sleep_interval = self.params.get('sleep_interval')
 381             if min_sleep_interval:
 382                 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
 383                 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
 384                 self.to_screen(
 385                     '[download] Sleeping %s seconds ...' % (
 386                         int(sleep_interval) if sleep_interval.is_integer()
 387                         else '%.2f' % sleep_interval))
 388                 time.sleep(sleep_interval)
 389         else:
 390             sleep_interval_sub = 0
 391             if type(self.params.get('sleep_interval_subtitles')) is int:
 392                 sleep_interval_sub = self.params.get('sleep_interval_subtitles')
 393             if sleep_interval_sub > 0:
 394                 self.to_screen(
 395                     '[download] Sleeping %s seconds ...' % (
 396                         sleep_interval_sub))
 397                 time.sleep(sleep_interval_sub)
 398         ret = self.real_download(filename, info_dict)
 399         self._finish_multiline_status()
 400         return ret, True
 401
 402     def real_download(self, filename, info_dict):
 403         """Real download process. Redefine in subclasses."""
 404         raise NotImplementedError('This method must be implemented by subclasses')
 405
 406     def _hook_progress(self, status, info_dict):
 407         if not self._progress_hooks:
 408             return
 409         info_dict = dict(info_dict)
 410         for key in ('__original_infodict', '__postprocessors'):
 411             info_dict.pop(key, None)
 412         # youtube-dl passes the same status object to all the hooks.
 413         # Some third party scripts seems to be relying on this.
 414         # So keep this behavior if possible
 415         status['info_dict'] = copy.deepcopy(info_dict)
 416         for ph in self._progress_hooks:
 417             ph(status)
 418
 419     def add_progress_hook(self, ph):
 420         # See YoutubeDl.py (search for progress_hooks) for a description of
 421         # this interface
 422         self._progress_hooks.append(ph)
 423
 424     def _debug_cmd(self, args, exe=None):
 425         if not self.params.get('verbose', False):
 426             return
 427
 428         str_args = [decodeArgument(a) for a in args]
 429
 430         if exe is None:
 431             exe = os.path.basename(str_args[0])
 432
 433         self.write_debug('%s command line: %s' % (exe, shell_quote(str_args)))