yt_dlp/downloader/common.py

   1 import contextlib
   2 import errno
   3 import os
   4 import random
   5 import re
   6 import time
   7
   8 from ..minicurses import (
   9     BreaklineStatusPrinter,
  10     MultilineLogger,
  11     MultilinePrinter,
  12     QuietMultilinePrinter,
  13 )
  14 from ..utils import (
  15     NUMBER_RE,
  16     LockingUnsupportedError,
  17     Namespace,
  18     classproperty,
  19     decodeArgument,
  20     encodeFilename,
  21     error_to_compat_str,
  22     float_or_none,
  23     format_bytes,
  24     join_nonempty,
  25     sanitize_open,
  26     shell_quote,
  27     timeconvert,
  28     timetuple_from_msec,
  29     try_call,
  30 )
  31
  32
  33 class FileDownloader:
  34     """File Downloader class.
  35
  36     File downloader objects are the ones responsible of downloading the
  37     actual video file and writing it to disk.
  38
  39     File downloaders accept a lot of parameters. In order not to saturate
  40     the object constructor with arguments, it receives a dictionary of
  41     options instead.
  42
  43     Available options:
  44
  45     verbose:            Print additional info to stdout.
  46     quiet:              Do not print messages to stdout.
  47     ratelimit:          Download speed limit, in bytes/sec.
  48     continuedl:         Attempt to continue downloads if possible
  49     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
  50     retries:            Number of times to retry for HTTP error 5xx
  51     file_access_retries:   Number of times to retry on file access error
  52     buffersize:         Size of download buffer in bytes.
  53     noresizebuffer:     Do not automatically resize the download buffer.
  54     continuedl:         Try to continue downloads if possible.
  55     noprogress:         Do not print the progress bar.
  56     nopart:             Do not use temporary .part files.
  57     updatetime:         Use the Last-modified header to set output file timestamps.
  58     test:               Download only first bytes to test the downloader.
  59     min_filesize:       Skip files smaller than this size
  60     max_filesize:       Skip files larger than this size
  61     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
  62     external_downloader_args:  A dictionary of downloader keys (in lower case)
  63                         and a list of additional command-line arguments for the
  64                         executable. Use 'default' as the name for arguments to be
  65                         passed to all downloaders. For compatibility with youtube-dl,
  66                         a single list of args can also be used
  67     hls_use_mpegts:     Use the mpegts container for HLS videos.
  68     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
  69                         useful for bypassing bandwidth throttling imposed by
  70                         a webserver (experimental)
  71     progress_template:  See YoutubeDL.py
  72     retry_sleep_functions: See YoutubeDL.py
  73
  74     Subclasses of this one must re-define the real_download method.
  75     """
  76
  77     _TEST_FILE_SIZE = 10241
  78     params = None
  79
  80     def __init__(self, ydl, params):
  81         """Create a FileDownloader object with the given options."""
  82         self._set_ydl(ydl)
  83         self._progress_hooks = []
  84         self.params = params
  85         self._prepare_multiline_status()
  86         self.add_progress_hook(self.report_progress)
  87
  88     def _set_ydl(self, ydl):
  89         self.ydl = ydl
  90
  91         for func in (
  92             'deprecation_warning',
  93             'report_error',
  94             'report_file_already_downloaded',
  95             'report_warning',
  96             'to_console_title',
  97             'to_stderr',
  98             'trouble',
  99             'write_debug',
 100         ):
 101             if not hasattr(self, func):
 102                 setattr(self, func, getattr(ydl, func))
 103
 104     def to_screen(self, *args, **kargs):
 105         self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs)
 106
 107     __to_screen = to_screen
 108
 109     @classproperty
 110     def FD_NAME(cls):
 111         return re.sub(r'(?<!^)(?=[A-Z])', '_', cls.__name__[:-2]).lower()
 112
 113     @staticmethod
 114     def format_seconds(seconds):
 115         if seconds is None:
 116             return ' Unknown'
 117         time = timetuple_from_msec(seconds * 1000)
 118         if time.hours > 99:
 119             return '--:--:--'
 120         if not time.hours:
 121             return '%02d:%02d' % time[1:-1]
 122         return '%02d:%02d:%02d' % time[:-1]
 123
 124     format_eta = format_seconds
 125
 126     @staticmethod
 127     def calc_percent(byte_counter, data_len):
 128         if data_len is None:
 129             return None
 130         return float(byte_counter) / float(data_len) * 100.0
 131
 132     @staticmethod
 133     def format_percent(percent):
 134         return '  N/A%' if percent is None else f'{percent:>5.1f}%'
 135
 136     @staticmethod
 137     def calc_eta(start, now, total, current):
 138         if total is None:
 139             return None
 140         if now is None:
 141             now = time.time()
 142         dif = now - start
 143         if current == 0 or dif < 0.001:  # One millisecond
 144             return None
 145         rate = float(current) / dif
 146         return int((float(total) - float(current)) / rate)
 147
 148     @staticmethod
 149     def calc_speed(start, now, bytes):
 150         dif = now - start
 151         if bytes == 0 or dif < 0.001:  # One millisecond
 152             return None
 153         return float(bytes) / dif
 154
 155     @staticmethod
 156     def format_speed(speed):
 157         return ' Unknown B/s' if speed is None else f'{format_bytes(speed):>10s}/s'
 158
 159     @staticmethod
 160     def format_retries(retries):
 161         return 'inf' if retries == float('inf') else int(retries)
 162
 163     @staticmethod
 164     def best_block_size(elapsed_time, bytes):
 165         new_min = max(bytes / 2.0, 1.0)
 166         new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
 167         if elapsed_time < 0.001:
 168             return int(new_max)
 169         rate = bytes / elapsed_time
 170         if rate > new_max:
 171             return int(new_max)
 172         if rate < new_min:
 173             return int(new_min)
 174         return int(rate)
 175
 176     @staticmethod
 177     def parse_bytes(bytestr):
 178         """Parse a string indicating a byte quantity into an integer."""
 179         matchobj = re.match(rf'(?i)^({NUMBER_RE})([kMGTPEZY]?)$', bytestr)
 180         if matchobj is None:
 181             return None
 182         number = float(matchobj.group(1))
 183         multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 184         return int(round(number * multiplier))
 185
 186     def slow_down(self, start_time, now, byte_counter):
 187         """Sleep if the download speed is over the rate limit."""
 188         rate_limit = self.params.get('ratelimit')
 189         if rate_limit is None or byte_counter == 0:
 190             return
 191         if now is None:
 192             now = time.time()
 193         elapsed = now - start_time
 194         if elapsed <= 0.0:
 195             return
 196         speed = float(byte_counter) / elapsed
 197         if speed > rate_limit:
 198             sleep_time = float(byte_counter) / rate_limit - elapsed
 199             if sleep_time > 0:
 200                 time.sleep(sleep_time)
 201
 202     def temp_name(self, filename):
 203         """Returns a temporary filename for the given filename."""
 204         if self.params.get('nopart', False) or filename == '-' or \
 205                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 206             return filename
 207         return filename + '.part'
 208
 209     def undo_temp_name(self, filename):
 210         if filename.endswith('.part'):
 211             return filename[:-len('.part')]
 212         return filename
 213
 214     def ytdl_filename(self, filename):
 215         return filename + '.ytdl'
 216
 217     def wrap_file_access(action, *, fatal=False):
 218         def outer(func):
 219             def inner(self, *args, **kwargs):
 220                 file_access_retries = self.params.get('file_access_retries', 0)
 221                 retry = 0
 222                 while True:
 223                     try:
 224                         return func(self, *args, **kwargs)
 225                     except OSError as err:
 226                         retry = retry + 1
 227                         if retry > file_access_retries or err.errno not in (errno.EACCES, errno.EINVAL):
 228                             if not fatal:
 229                                 self.report_error(f'unable to {action} file: {err}')
 230                                 return
 231                             raise
 232                         self.to_screen(
 233                             f'[download] Unable to {action} file due to file access error. '
 234                             f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...')
 235                         if not self.sleep_retry('file_access', retry):
 236                             time.sleep(0.01)
 237             return inner
 238         return outer
 239
 240     @wrap_file_access('open', fatal=True)
 241     def sanitize_open(self, filename, open_mode):
 242         f, filename = sanitize_open(filename, open_mode)
 243         if not getattr(f, 'locked', None):
 244             self.write_debug(f'{LockingUnsupportedError.msg}. Proceeding without locking', only_once=True)
 245         return f, filename
 246
 247     @wrap_file_access('remove')
 248     def try_remove(self, filename):
 249         os.remove(filename)
 250
 251     @wrap_file_access('rename')
 252     def try_rename(self, old_filename, new_filename):
 253         if old_filename == new_filename:
 254             return
 255         os.replace(old_filename, new_filename)
 256
 257     def try_utime(self, filename, last_modified_hdr):
 258         """Try to set the last-modified time of the given file."""
 259         if last_modified_hdr is None:
 260             return
 261         if not os.path.isfile(encodeFilename(filename)):
 262             return
 263         timestr = last_modified_hdr
 264         if timestr is None:
 265             return
 266         filetime = timeconvert(timestr)
 267         if filetime is None:
 268             return filetime
 269         # Ignore obviously invalid dates
 270         if filetime == 0:
 271             return
 272         with contextlib.suppress(Exception):
 273             os.utime(filename, (time.time(), filetime))
 274         return filetime
 275
 276     def report_destination(self, filename):
 277         """Report destination filename."""
 278         self.to_screen('[download] Destination: ' + filename)
 279
 280     def _prepare_multiline_status(self, lines=1):
 281         if self.params.get('noprogress'):
 282             self._multiline = QuietMultilinePrinter()
 283         elif self.ydl.params.get('logger'):
 284             self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
 285         elif self.params.get('progress_with_newline'):
 286             self._multiline = BreaklineStatusPrinter(self.ydl._out_files.out, lines)
 287         else:
 288             self._multiline = MultilinePrinter(self.ydl._out_files.out, lines, not self.params.get('quiet'))
 289         self._multiline.allow_colors = self._multiline._HAVE_FULLCAP and not self.params.get('no_color')
 290
 291     def _finish_multiline_status(self):
 292         self._multiline.end()
 293
 294     ProgressStyles = Namespace(
 295         downloaded_bytes='light blue',
 296         percent='light blue',
 297         eta='yellow',
 298         speed='green',
 299         elapsed='bold white',
 300         total_bytes='',
 301         total_bytes_estimate='',
 302     )
 303
 304     def _report_progress_status(self, s, default_template):
 305         for name, style in self.ProgressStyles:
 306             name = f'_{name}_str'
 307             if name not in s:
 308                 continue
 309             s[name] = self._format_progress(s[name], style)
 310         s['_default_template'] = default_template % s
 311
 312         progress_dict = s.copy()
 313         progress_dict.pop('info_dict')
 314         progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
 315
 316         progress_template = self.params.get('progress_template', {})
 317         self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
 318             progress_template.get('download') or '[download] %(progress._default_template)s',
 319             progress_dict), s.get('progress_idx') or 0)
 320         self.to_console_title(self.ydl.evaluate_outtmpl(
 321             progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
 322             progress_dict))
 323
 324     def _format_progress(self, *args, **kwargs):
 325         return self.ydl._format_text(
 326             self._multiline.stream, self._multiline.allow_colors, *args, **kwargs)
 327
 328     def report_progress(self, s):
 329         def with_fields(*tups, default=''):
 330             for *fields, tmpl in tups:
 331                 if all(s.get(f) is not None for f in fields):
 332                     return tmpl
 333             return default
 334
 335         if s['status'] == 'finished':
 336             if self.params.get('noprogress'):
 337                 self.to_screen('[download] Download completed')
 338             s.update({
 339                 '_total_bytes_str': format_bytes(s.get('total_bytes')),
 340                 '_elapsed_str': self.format_seconds(s.get('elapsed')),
 341                 '_percent_str': self.format_percent(100),
 342             })
 343             self._report_progress_status(s, join_nonempty(
 344                 '100%%',
 345                 with_fields(('total_bytes', 'of %(_total_bytes_str)s')),
 346                 with_fields(('elapsed', 'in %(_elapsed_str)s')),
 347                 delim=' '))
 348
 349         if s['status'] != 'downloading':
 350             return
 351
 352         s.update({
 353             '_eta_str': self.format_eta(s.get('eta')),
 354             '_speed_str': self.format_speed(s.get('speed')),
 355             '_percent_str': self.format_percent(try_call(
 356                 lambda: 100 * s['downloaded_bytes'] / s['total_bytes'],
 357                 lambda: 100 * s['downloaded_bytes'] / s['total_bytes_estimate'],
 358                 lambda: s['downloaded_bytes'] == 0 and 0)),
 359             '_total_bytes_str': format_bytes(s.get('total_bytes')),
 360             '_total_bytes_estimate_str': format_bytes(s.get('total_bytes_estimate')),
 361             '_downloaded_bytes_str': format_bytes(s.get('downloaded_bytes')),
 362             '_elapsed_str': self.format_seconds(s.get('elapsed')),
 363         })
 364
 365         msg_template = with_fields(
 366             ('total_bytes', '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'),
 367             ('total_bytes_estimate', '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'),
 368             ('downloaded_bytes', 'elapsed', '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'),
 369             ('downloaded_bytes', '%(_downloaded_bytes_str)s at %(_speed_str)s'),
 370             default='%(_percent_str)s at %(_speed_str)s ETA %(_eta_str)s')
 371
 372         msg_template += with_fields(
 373             ('fragment_index', 'fragment_count', ' (frag %(fragment_index)s/%(fragment_count)s)'),
 374             ('fragment_index', ' (frag %(fragment_index)s)'))
 375         self._report_progress_status(s, msg_template)
 376
 377     def report_resuming_byte(self, resume_len):
 378         """Report attempt to resume at given byte."""
 379         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 380
 381     def report_retry(self, err, count, retries):
 382         """Report retry in case of HTTP error 5xx"""
 383         self.__to_screen(
 384             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
 385             % (error_to_compat_str(err), count, self.format_retries(retries)))
 386         self.sleep_retry('http', count)
 387
 388     def report_unable_to_resume(self):
 389         """Report it was impossible to resume download."""
 390         self.to_screen('[download] Unable to resume')
 391
 392     def sleep_retry(self, retry_type, count):
 393         sleep_func = self.params.get('retry_sleep_functions', {}).get(retry_type)
 394         delay = float_or_none(sleep_func(n=count - 1)) if sleep_func else None
 395         if delay:
 396             self.__to_screen(f'Sleeping {delay:.2f} seconds ...')
 397             time.sleep(delay)
 398         return sleep_func is not None
 399
 400     @staticmethod
 401     def supports_manifest(manifest):
 402         """ Whether the downloader can download the fragments from the manifest.
 403         Redefine in subclasses if needed. """
 404         pass
 405
 406     def download(self, filename, info_dict, subtitle=False):
 407         """Download to a filename using the info from info_dict
 408         Return True on success and False otherwise
 409         """
 410
 411         nooverwrites_and_exists = (
 412             not self.params.get('overwrites', True)
 413             and os.path.exists(encodeFilename(filename))
 414         )
 415
 416         if not hasattr(filename, 'write'):
 417             continuedl_and_exists = (
 418                 self.params.get('continuedl', True)
 419                 and os.path.isfile(encodeFilename(filename))
 420                 and not self.params.get('nopart', False)
 421             )
 422
 423             # Check file already present
 424             if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
 425                 self.report_file_already_downloaded(filename)
 426                 self._hook_progress({
 427                     'filename': filename,
 428                     'status': 'finished',
 429                     'total_bytes': os.path.getsize(encodeFilename(filename)),
 430                 }, info_dict)
 431                 self._finish_multiline_status()
 432                 return True, False
 433
 434         if subtitle:
 435             sleep_interval = self.params.get('sleep_interval_subtitles') or 0
 436         else:
 437             min_sleep_interval = self.params.get('sleep_interval') or 0
 438             sleep_interval = random.uniform(
 439                 min_sleep_interval, self.params.get('max_sleep_interval') or min_sleep_interval)
 440         if sleep_interval > 0:
 441             self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds ...')
 442             time.sleep(sleep_interval)
 443
 444         ret = self.real_download(filename, info_dict)
 445         self._finish_multiline_status()
 446         return ret, True
 447
 448     def real_download(self, filename, info_dict):
 449         """Real download process. Redefine in subclasses."""
 450         raise NotImplementedError('This method must be implemented by subclasses')
 451
 452     def _hook_progress(self, status, info_dict):
 453         if not self._progress_hooks:
 454             return
 455         status['info_dict'] = info_dict
 456         # youtube-dl passes the same status object to all the hooks.
 457         # Some third party scripts seems to be relying on this.
 458         # So keep this behavior if possible
 459         for ph in self._progress_hooks:
 460             ph(status)
 461
 462     def add_progress_hook(self, ph):
 463         # See YoutubeDl.py (search for progress_hooks) for a description of
 464         # this interface
 465         self._progress_hooks.append(ph)
 466
 467     def _debug_cmd(self, args, exe=None):
 468         if not self.params.get('verbose', False):
 469             return
 470
 471         str_args = [decodeArgument(a) for a in args]
 472
 473         if exe is None:
 474             exe = os.path.basename(str_args[0])
 475
 476         self.write_debug(f'{exe} command line: {shell_quote(str_args)}')