yt_dlp/downloader/common.py

   1 import errno
   2 import os
   3 import random
   4 import re
   5 import time
   6
   7 from ..minicurses import (
   8     BreaklineStatusPrinter,
   9     MultilineLogger,
  10     MultilinePrinter,
  11     QuietMultilinePrinter,
  12 )
  13 from ..utils import (
  14     LockingUnsupportedError,
  15     decodeArgument,
  16     encodeFilename,
  17     error_to_compat_str,
  18     format_bytes,
  19     sanitize_open,
  20     shell_quote,
  21     timeconvert,
  22     timetuple_from_msec,
  23 )
  24
  25
  26 class FileDownloader:
  27     """File Downloader class.
  28
  29     File downloader objects are the ones responsible of downloading the
  30     actual video file and writing it to disk.
  31
  32     File downloaders accept a lot of parameters. In order not to saturate
  33     the object constructor with arguments, it receives a dictionary of
  34     options instead.
  35
  36     Available options:
  37
  38     verbose:            Print additional info to stdout.
  39     quiet:              Do not print messages to stdout.
  40     ratelimit:          Download speed limit, in bytes/sec.
  41     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
  42     retries:            Number of times to retry for HTTP error 5xx
  43     file_access_retries:   Number of times to retry on file access error
  44     buffersize:         Size of download buffer in bytes.
  45     noresizebuffer:     Do not automatically resize the download buffer.
  46     continuedl:         Try to continue downloads if possible.
  47     noprogress:         Do not print the progress bar.
  48     nopart:             Do not use temporary .part files.
  49     updatetime:         Use the Last-modified header to set output file timestamps.
  50     test:               Download only first bytes to test the downloader.
  51     min_filesize:       Skip files smaller than this size
  52     max_filesize:       Skip files larger than this size
  53     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
  54     external_downloader_args:  A dictionary of downloader keys (in lower case)
  55                         and a list of additional command-line arguments for the
  56                         executable. Use 'default' as the name for arguments to be
  57                         passed to all downloaders. For compatibility with youtube-dl,
  58                         a single list of args can also be used
  59     hls_use_mpegts:     Use the mpegts container for HLS videos.
  60     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
  61                         useful for bypassing bandwidth throttling imposed by
  62                         a webserver (experimental)
  63     progress_template:  See YoutubeDL.py
  64
  65     Subclasses of this one must re-define the real_download method.
  66     """
  67
  68     _TEST_FILE_SIZE = 10241
  69     params = None
  70
  71     def __init__(self, ydl, params):
  72         """Create a FileDownloader object with the given options."""
  73         self.ydl = ydl
  74         self._progress_hooks = []
  75         self.params = params
  76         self._prepare_multiline_status()
  77         self.add_progress_hook(self.report_progress)
  78
  79     @staticmethod
  80     def format_seconds(seconds):
  81         time = timetuple_from_msec(seconds * 1000)
  82         if time.hours > 99:
  83             return '--:--:--'
  84         if not time.hours:
  85             return '%02d:%02d' % time[1:-1]
  86         return '%02d:%02d:%02d' % time[:-1]
  87
  88     @staticmethod
  89     def calc_percent(byte_counter, data_len):
  90         if data_len is None:
  91             return None
  92         return float(byte_counter) / float(data_len) * 100.0
  93
  94     @staticmethod
  95     def format_percent(percent):
  96         if percent is None:
  97             return '---.-%'
  98         elif percent == 100:
  99             return '100%'
 100         return '%6s' % ('%3.1f%%' % percent)
 101
 102     @staticmethod
 103     def calc_eta(start, now, total, current):
 104         if total is None:
 105             return None
 106         if now is None:
 107             now = time.time()
 108         dif = now - start
 109         if current == 0 or dif < 0.001:  # One millisecond
 110             return None
 111         rate = float(current) / dif
 112         return int((float(total) - float(current)) / rate)
 113
 114     @staticmethod
 115     def format_eta(eta):
 116         if eta is None:
 117             return '--:--'
 118         return FileDownloader.format_seconds(eta)
 119
 120     @staticmethod
 121     def calc_speed(start, now, bytes):
 122         dif = now - start
 123         if bytes == 0 or dif < 0.001:  # One millisecond
 124             return None
 125         return float(bytes) / dif
 126
 127     @staticmethod
 128     def format_speed(speed):
 129         if speed is None:
 130             return '%10s' % '---b/s'
 131         return '%10s' % ('%s/s' % format_bytes(speed))
 132
 133     @staticmethod
 134     def format_retries(retries):
 135         return 'inf' if retries == float('inf') else '%.0f' % retries
 136
 137     @staticmethod
 138     def best_block_size(elapsed_time, bytes):
 139         new_min = max(bytes / 2.0, 1.0)
 140         new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
 141         if elapsed_time < 0.001:
 142             return int(new_max)
 143         rate = bytes / elapsed_time
 144         if rate > new_max:
 145             return int(new_max)
 146         if rate < new_min:
 147             return int(new_min)
 148         return int(rate)
 149
 150     @staticmethod
 151     def parse_bytes(bytestr):
 152         """Parse a string indicating a byte quantity into an integer."""
 153         matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 154         if matchobj is None:
 155             return None
 156         number = float(matchobj.group(1))
 157         multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 158         return int(round(number * multiplier))
 159
 160     def to_screen(self, *args, **kargs):
 161         self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs)
 162
 163     def to_stderr(self, message):
 164         self.ydl.to_stderr(message)
 165
 166     def to_console_title(self, message):
 167         self.ydl.to_console_title(message)
 168
 169     def trouble(self, *args, **kargs):
 170         self.ydl.trouble(*args, **kargs)
 171
 172     def report_warning(self, *args, **kargs):
 173         self.ydl.report_warning(*args, **kargs)
 174
 175     def report_error(self, *args, **kargs):
 176         self.ydl.report_error(*args, **kargs)
 177
 178     def write_debug(self, *args, **kargs):
 179         self.ydl.write_debug(*args, **kargs)
 180
 181     def slow_down(self, start_time, now, byte_counter):
 182         """Sleep if the download speed is over the rate limit."""
 183         rate_limit = self.params.get('ratelimit')
 184         if rate_limit is None or byte_counter == 0:
 185             return
 186         if now is None:
 187             now = time.time()
 188         elapsed = now - start_time
 189         if elapsed <= 0.0:
 190             return
 191         speed = float(byte_counter) / elapsed
 192         if speed > rate_limit:
 193             sleep_time = float(byte_counter) / rate_limit - elapsed
 194             if sleep_time > 0:
 195                 time.sleep(sleep_time)
 196
 197     def temp_name(self, filename):
 198         """Returns a temporary filename for the given filename."""
 199         if self.params.get('nopart', False) or filename == '-' or \
 200                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 201             return filename
 202         return filename + '.part'
 203
 204     def undo_temp_name(self, filename):
 205         if filename.endswith('.part'):
 206             return filename[:-len('.part')]
 207         return filename
 208
 209     def ytdl_filename(self, filename):
 210         return filename + '.ytdl'
 211
 212     def wrap_file_access(action, *, fatal=False):
 213         def outer(func):
 214             def inner(self, *args, **kwargs):
 215                 file_access_retries = self.params.get('file_access_retries', 0)
 216                 retry = 0
 217                 while True:
 218                     try:
 219                         return func(self, *args, **kwargs)
 220                     except OSError as err:
 221                         retry = retry + 1
 222                         if retry > file_access_retries or err.errno not in (errno.EACCES, errno.EINVAL):
 223                             if not fatal:
 224                                 self.report_error(f'unable to {action} file: {err}')
 225                                 return
 226                             raise
 227                         self.to_screen(
 228                             f'[download] Unable to {action} file due to file access error. '
 229                             f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...')
 230                         time.sleep(0.01)
 231             return inner
 232         return outer
 233
 234     @wrap_file_access('open', fatal=True)
 235     def sanitize_open(self, filename, open_mode):
 236         f, filename = sanitize_open(filename, open_mode)
 237         if not getattr(f, 'locked', None):
 238             self.write_debug(f'{LockingUnsupportedError.msg}. Proceeding without locking', only_once=True)
 239         return f, filename
 240
 241     @wrap_file_access('remove')
 242     def try_remove(self, filename):
 243         os.remove(filename)
 244
 245     @wrap_file_access('rename')
 246     def try_rename(self, old_filename, new_filename):
 247         if old_filename == new_filename:
 248             return
 249         os.replace(old_filename, new_filename)
 250
 251     def try_utime(self, filename, last_modified_hdr):
 252         """Try to set the last-modified time of the given file."""
 253         if last_modified_hdr is None:
 254             return
 255         if not os.path.isfile(encodeFilename(filename)):
 256             return
 257         timestr = last_modified_hdr
 258         if timestr is None:
 259             return
 260         filetime = timeconvert(timestr)
 261         if filetime is None:
 262             return filetime
 263         # Ignore obviously invalid dates
 264         if filetime == 0:
 265             return
 266         try:
 267             os.utime(filename, (time.time(), filetime))
 268         except Exception:
 269             pass
 270         return filetime
 271
 272     def report_destination(self, filename):
 273         """Report destination filename."""
 274         self.to_screen('[download] Destination: ' + filename)
 275
 276     def _prepare_multiline_status(self, lines=1):
 277         if self.params.get('noprogress'):
 278             self._multiline = QuietMultilinePrinter()
 279         elif self.ydl.params.get('logger'):
 280             self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
 281         elif self.params.get('progress_with_newline'):
 282             self._multiline = BreaklineStatusPrinter(self.ydl._out_files['screen'], lines)
 283         else:
 284             self._multiline = MultilinePrinter(self.ydl._out_files['screen'], lines, not self.params.get('quiet'))
 285         self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color')
 286
 287     def _finish_multiline_status(self):
 288         self._multiline.end()
 289
 290     _progress_styles = {
 291         'downloaded_bytes': 'light blue',
 292         'percent': 'light blue',
 293         'eta': 'yellow',
 294         'speed': 'green',
 295         'elapsed': 'bold white',
 296         'total_bytes': '',
 297         'total_bytes_estimate': '',
 298     }
 299
 300     def _report_progress_status(self, s, default_template):
 301         for name, style in self._progress_styles.items():
 302             name = f'_{name}_str'
 303             if name not in s:
 304                 continue
 305             s[name] = self._format_progress(s[name], style)
 306         s['_default_template'] = default_template % s
 307
 308         progress_dict = s.copy()
 309         progress_dict.pop('info_dict')
 310         progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
 311
 312         progress_template = self.params.get('progress_template', {})
 313         self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
 314             progress_template.get('download') or '[download] %(progress._default_template)s',
 315             progress_dict), s.get('progress_idx') or 0)
 316         self.to_console_title(self.ydl.evaluate_outtmpl(
 317             progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
 318             progress_dict))
 319
 320     def _format_progress(self, *args, **kwargs):
 321         return self.ydl._format_text(
 322             self._multiline.stream, self._multiline.allow_colors, *args, **kwargs)
 323
 324     def report_progress(self, s):
 325         if s['status'] == 'finished':
 326             if self.params.get('noprogress'):
 327                 self.to_screen('[download] Download completed')
 328             msg_template = '100%%'
 329             if s.get('total_bytes') is not None:
 330                 s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 331                 msg_template += ' of %(_total_bytes_str)s'
 332             if s.get('elapsed') is not None:
 333                 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 334                 msg_template += ' in %(_elapsed_str)s'
 335             s['_percent_str'] = self.format_percent(100)
 336             self._report_progress_status(s, msg_template)
 337             return
 338
 339         if s['status'] != 'downloading':
 340             return
 341
 342         if s.get('eta') is not None:
 343             s['_eta_str'] = self.format_eta(s['eta'])
 344         else:
 345             s['_eta_str'] = 'Unknown'
 346
 347         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
 348             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
 349         elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
 350             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
 351         else:
 352             if s.get('downloaded_bytes') == 0:
 353                 s['_percent_str'] = self.format_percent(0)
 354             else:
 355                 s['_percent_str'] = 'Unknown %'
 356
 357         if s.get('speed') is not None:
 358             s['_speed_str'] = self.format_speed(s['speed'])
 359         else:
 360             s['_speed_str'] = 'Unknown speed'
 361
 362         if s.get('total_bytes') is not None:
 363             s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 364             msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
 365         elif s.get('total_bytes_estimate') is not None:
 366             s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
 367             msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
 368         else:
 369             if s.get('downloaded_bytes') is not None:
 370                 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
 371                 if s.get('elapsed'):
 372                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 373                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
 374                 else:
 375                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
 376             else:
 377                 msg_template = '%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s'
 378         if s.get('fragment_index') and s.get('fragment_count'):
 379             msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)'
 380         elif s.get('fragment_index'):
 381             msg_template += ' (frag %(fragment_index)s)'
 382         self._report_progress_status(s, msg_template)
 383
 384     def report_resuming_byte(self, resume_len):
 385         """Report attempt to resume at given byte."""
 386         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 387
 388     def report_retry(self, err, count, retries):
 389         """Report retry in case of HTTP error 5xx"""
 390         self.to_screen(
 391             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
 392             % (error_to_compat_str(err), count, self.format_retries(retries)))
 393
 394     def report_file_already_downloaded(self, *args, **kwargs):
 395         """Report file has already been fully downloaded."""
 396         return self.ydl.report_file_already_downloaded(*args, **kwargs)
 397
 398     def report_unable_to_resume(self):
 399         """Report it was impossible to resume download."""
 400         self.to_screen('[download] Unable to resume')
 401
 402     @staticmethod
 403     def supports_manifest(manifest):
 404         """ Whether the downloader can download the fragments from the manifest.
 405         Redefine in subclasses if needed. """
 406         pass
 407
 408     def download(self, filename, info_dict, subtitle=False):
 409         """Download to a filename using the info from info_dict
 410         Return True on success and False otherwise
 411         """
 412
 413         nooverwrites_and_exists = (
 414             not self.params.get('overwrites', True)
 415             and os.path.exists(encodeFilename(filename))
 416         )
 417
 418         if not hasattr(filename, 'write'):
 419             continuedl_and_exists = (
 420                 self.params.get('continuedl', True)
 421                 and os.path.isfile(encodeFilename(filename))
 422                 and not self.params.get('nopart', False)
 423             )
 424
 425             # Check file already present
 426             if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
 427                 self.report_file_already_downloaded(filename)
 428                 self._hook_progress({
 429                     'filename': filename,
 430                     'status': 'finished',
 431                     'total_bytes': os.path.getsize(encodeFilename(filename)),
 432                 }, info_dict)
 433                 self._finish_multiline_status()
 434                 return True, False
 435
 436         if subtitle is False:
 437             min_sleep_interval = self.params.get('sleep_interval')
 438             if min_sleep_interval:
 439                 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
 440                 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
 441                 self.to_screen(
 442                     '[download] Sleeping %s seconds ...' % (
 443                         int(sleep_interval) if sleep_interval.is_integer()
 444                         else '%.2f' % sleep_interval))
 445                 time.sleep(sleep_interval)
 446         else:
 447             sleep_interval_sub = 0
 448             if type(self.params.get('sleep_interval_subtitles')) is int:
 449                 sleep_interval_sub = self.params.get('sleep_interval_subtitles')
 450             if sleep_interval_sub > 0:
 451                 self.to_screen(
 452                     '[download] Sleeping %s seconds ...' % (
 453                         sleep_interval_sub))
 454                 time.sleep(sleep_interval_sub)
 455         ret = self.real_download(filename, info_dict)
 456         self._finish_multiline_status()
 457         return ret, True
 458
 459     def real_download(self, filename, info_dict):
 460         """Real download process. Redefine in subclasses."""
 461         raise NotImplementedError('This method must be implemented by subclasses')
 462
 463     def _hook_progress(self, status, info_dict):
 464         if not self._progress_hooks:
 465             return
 466         status['info_dict'] = info_dict
 467         # youtube-dl passes the same status object to all the hooks.
 468         # Some third party scripts seems to be relying on this.
 469         # So keep this behavior if possible
 470         for ph in self._progress_hooks:
 471             ph(status)
 472
 473     def add_progress_hook(self, ph):
 474         # See YoutubeDl.py (search for progress_hooks) for a description of
 475         # this interface
 476         self._progress_hooks.append(ph)
 477
 478     def _debug_cmd(self, args, exe=None):
 479         if not self.params.get('verbose', False):
 480             return
 481
 482         str_args = [decodeArgument(a) for a in args]
 483
 484         if exe is None:
 485             exe = os.path.basename(str_args[0])
 486
 487         self.write_debug(f'{exe} command line: {shell_quote(str_args)}')