yt_dlp/downloader/common.py

   1 from __future__ import division, unicode_literals
   2
   3 import os
   4 import re
   5 import time
   6 import random
   7
   8 from ..utils import (
   9     decodeArgument,
  10     encodeFilename,
  11     error_to_compat_str,
  12     format_bytes,
  13     shell_quote,
  14     timeconvert,
  15     timetuple_from_msec,
  16 )
  17 from ..minicurses import (
  18     MultilineLogger,
  19     MultilinePrinter,
  20     QuietMultilinePrinter,
  21     BreaklineStatusPrinter
  22 )
  23
  24
  25 class FileDownloader(object):
  26     """File Downloader class.
  27
  28     File downloader objects are the ones responsible of downloading the
  29     actual video file and writing it to disk.
  30
  31     File downloaders accept a lot of parameters. In order not to saturate
  32     the object constructor with arguments, it receives a dictionary of
  33     options instead.
  34
  35     Available options:
  36
  37     verbose:            Print additional info to stdout.
  38     quiet:              Do not print messages to stdout.
  39     ratelimit:          Download speed limit, in bytes/sec.
  40     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
  41     retries:            Number of times to retry for HTTP error 5xx
  42     buffersize:         Size of download buffer in bytes.
  43     noresizebuffer:     Do not automatically resize the download buffer.
  44     continuedl:         Try to continue downloads if possible.
  45     noprogress:         Do not print the progress bar.
  46     nopart:             Do not use temporary .part files.
  47     updatetime:         Use the Last-modified header to set output file timestamps.
  48     test:               Download only first bytes to test the downloader.
  49     min_filesize:       Skip files smaller than this size
  50     max_filesize:       Skip files larger than this size
  51     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
  52     external_downloader_args:  A dictionary of downloader keys (in lower case)
  53                         and a list of additional command-line arguments for the
  54                         executable. Use 'default' as the name for arguments to be
  55                         passed to all downloaders. For compatibility with youtube-dl,
  56                         a single list of args can also be used
  57     hls_use_mpegts:     Use the mpegts container for HLS videos.
  58     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
  59                         useful for bypassing bandwidth throttling imposed by
  60                         a webserver (experimental)
  61     progress_template:  See YoutubeDL.py
  62
  63     Subclasses of this one must re-define the real_download method.
  64     """
  65
  66     _TEST_FILE_SIZE = 10241
  67     params = None
  68
  69     def __init__(self, ydl, params):
  70         """Create a FileDownloader object with the given options."""
  71         self.ydl = ydl
  72         self._progress_hooks = []
  73         self.params = params
  74         self._prepare_multiline_status()
  75         self.add_progress_hook(self.report_progress)
  76
  77     @staticmethod
  78     def format_seconds(seconds):
  79         time = timetuple_from_msec(seconds * 1000)
  80         if time.hours > 99:
  81             return '--:--:--'
  82         if not time.hours:
  83             return '%02d:%02d' % time[1:-1]
  84         return '%02d:%02d:%02d' % time[:-1]
  85
  86     @staticmethod
  87     def calc_percent(byte_counter, data_len):
  88         if data_len is None:
  89             return None
  90         return float(byte_counter) / float(data_len) * 100.0
  91
  92     @staticmethod
  93     def format_percent(percent):
  94         if percent is None:
  95             return '---.-%'
  96         return '%6s' % ('%3.1f%%' % percent)
  97
  98     @staticmethod
  99     def calc_eta(start, now, total, current):
 100         if total is None:
 101             return None
 102         if now is None:
 103             now = time.time()
 104         dif = now - start
 105         if current == 0 or dif < 0.001:  # One millisecond
 106             return None
 107         rate = float(current) / dif
 108         return int((float(total) - float(current)) / rate)
 109
 110     @staticmethod
 111     def format_eta(eta):
 112         if eta is None:
 113             return '--:--'
 114         return FileDownloader.format_seconds(eta)
 115
 116     @staticmethod
 117     def calc_speed(start, now, bytes):
 118         dif = now - start
 119         if bytes == 0 or dif < 0.001:  # One millisecond
 120             return None
 121         return float(bytes) / dif
 122
 123     @staticmethod
 124     def format_speed(speed):
 125         if speed is None:
 126             return '%10s' % '---b/s'
 127         return '%10s' % ('%s/s' % format_bytes(speed))
 128
 129     @staticmethod
 130     def format_retries(retries):
 131         return 'inf' if retries == float('inf') else '%.0f' % retries
 132
 133     @staticmethod
 134     def best_block_size(elapsed_time, bytes):
 135         new_min = max(bytes / 2.0, 1.0)
 136         new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
 137         if elapsed_time < 0.001:
 138             return int(new_max)
 139         rate = bytes / elapsed_time
 140         if rate > new_max:
 141             return int(new_max)
 142         if rate < new_min:
 143             return int(new_min)
 144         return int(rate)
 145
 146     @staticmethod
 147     def parse_bytes(bytestr):
 148         """Parse a string indicating a byte quantity into an integer."""
 149         matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 150         if matchobj is None:
 151             return None
 152         number = float(matchobj.group(1))
 153         multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 154         return int(round(number * multiplier))
 155
 156     def to_screen(self, *args, **kargs):
 157         self.ydl.to_stdout(*args, quiet=self.params.get('quiet'), **kargs)
 158
 159     def to_stderr(self, message):
 160         self.ydl.to_stderr(message)
 161
 162     def to_console_title(self, message):
 163         self.ydl.to_console_title(message)
 164
 165     def trouble(self, *args, **kargs):
 166         self.ydl.trouble(*args, **kargs)
 167
 168     def report_warning(self, *args, **kargs):
 169         self.ydl.report_warning(*args, **kargs)
 170
 171     def report_error(self, *args, **kargs):
 172         self.ydl.report_error(*args, **kargs)
 173
 174     def write_debug(self, *args, **kargs):
 175         self.ydl.write_debug(*args, **kargs)
 176
 177     def slow_down(self, start_time, now, byte_counter):
 178         """Sleep if the download speed is over the rate limit."""
 179         rate_limit = self.params.get('ratelimit')
 180         if rate_limit is None or byte_counter == 0:
 181             return
 182         if now is None:
 183             now = time.time()
 184         elapsed = now - start_time
 185         if elapsed <= 0.0:
 186             return
 187         speed = float(byte_counter) / elapsed
 188         if speed > rate_limit:
 189             sleep_time = float(byte_counter) / rate_limit - elapsed
 190             if sleep_time > 0:
 191                 time.sleep(sleep_time)
 192
 193     def temp_name(self, filename):
 194         """Returns a temporary filename for the given filename."""
 195         if self.params.get('nopart', False) or filename == '-' or \
 196                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 197             return filename
 198         return filename + '.part'
 199
 200     def undo_temp_name(self, filename):
 201         if filename.endswith('.part'):
 202             return filename[:-len('.part')]
 203         return filename
 204
 205     def ytdl_filename(self, filename):
 206         return filename + '.ytdl'
 207
 208     def try_rename(self, old_filename, new_filename):
 209         if old_filename == new_filename:
 210             return
 211         try:
 212             os.replace(old_filename, new_filename)
 213         except (IOError, OSError) as err:
 214             self.report_error(f'unable to rename file: {err}')
 215
 216     def try_utime(self, filename, last_modified_hdr):
 217         """Try to set the last-modified time of the given file."""
 218         if last_modified_hdr is None:
 219             return
 220         if not os.path.isfile(encodeFilename(filename)):
 221             return
 222         timestr = last_modified_hdr
 223         if timestr is None:
 224             return
 225         filetime = timeconvert(timestr)
 226         if filetime is None:
 227             return filetime
 228         # Ignore obviously invalid dates
 229         if filetime == 0:
 230             return
 231         try:
 232             os.utime(filename, (time.time(), filetime))
 233         except Exception:
 234             pass
 235         return filetime
 236
 237     def report_destination(self, filename):
 238         """Report destination filename."""
 239         self.to_screen('[download] Destination: ' + filename)
 240
 241     def _prepare_multiline_status(self, lines=1):
 242         if self.params.get('noprogress'):
 243             self._multiline = QuietMultilinePrinter()
 244         elif self.ydl.params.get('logger'):
 245             self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
 246         elif self.params.get('progress_with_newline'):
 247             self._multiline = BreaklineStatusPrinter(self.ydl._screen_file, lines)
 248         else:
 249             self._multiline = MultilinePrinter(self.ydl._screen_file, lines, not self.params.get('quiet'))
 250
 251     def _finish_multiline_status(self):
 252         self._multiline.end()
 253
 254     def _report_progress_status(self, s):
 255         progress_dict = s.copy()
 256         progress_dict.pop('info_dict')
 257         progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
 258
 259         progress_template = self.params.get('progress_template', {})
 260         self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
 261             progress_template.get('download') or '[download] %(progress._default_template)s',
 262             progress_dict), s.get('progress_idx') or 0)
 263         self.to_console_title(self.ydl.evaluate_outtmpl(
 264             progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
 265             progress_dict))
 266
 267     def report_progress(self, s):
 268         if s['status'] == 'finished':
 269             if self.params.get('noprogress'):
 270                 self.to_screen('[download] Download completed')
 271             msg_template = '100%%'
 272             if s.get('total_bytes') is not None:
 273                 s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 274                 msg_template += ' of %(_total_bytes_str)s'
 275             if s.get('elapsed') is not None:
 276                 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 277                 msg_template += ' in %(_elapsed_str)s'
 278             s['_percent_str'] = self.format_percent(100)
 279             s['_default_template'] = msg_template % s
 280             self._report_progress_status(s)
 281             return
 282
 283         if s['status'] != 'downloading':
 284             return
 285
 286         if s.get('eta') is not None:
 287             s['_eta_str'] = self.format_eta(s['eta'])
 288         else:
 289             s['_eta_str'] = 'Unknown ETA'
 290
 291         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
 292             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
 293         elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
 294             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
 295         else:
 296             if s.get('downloaded_bytes') == 0:
 297                 s['_percent_str'] = self.format_percent(0)
 298             else:
 299                 s['_percent_str'] = 'Unknown %'
 300
 301         if s.get('speed') is not None:
 302             s['_speed_str'] = self.format_speed(s['speed'])
 303         else:
 304             s['_speed_str'] = 'Unknown speed'
 305
 306         if s.get('total_bytes') is not None:
 307             s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 308             msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
 309         elif s.get('total_bytes_estimate') is not None:
 310             s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
 311             msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
 312         else:
 313             if s.get('downloaded_bytes') is not None:
 314                 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
 315                 if s.get('elapsed'):
 316                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 317                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
 318                 else:
 319                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
 320             else:
 321                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
 322         if s.get('fragment_index') and s.get('fragment_count'):
 323             msg_template += ' (frag %(fragment_index)s/%(fragment_count)s)'
 324         elif s.get('fragment_index'):
 325             msg_template += ' (frag %(fragment_index)s)'
 326         s['_default_template'] = msg_template % s
 327         self._report_progress_status(s)
 328
 329     def report_resuming_byte(self, resume_len):
 330         """Report attempt to resume at given byte."""
 331         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 332
 333     def report_retry(self, err, count, retries):
 334         """Report retry in case of HTTP error 5xx"""
 335         self.to_screen(
 336             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
 337             % (error_to_compat_str(err), count, self.format_retries(retries)))
 338
 339     def report_file_already_downloaded(self, *args, **kwargs):
 340         """Report file has already been fully downloaded."""
 341         return self.ydl.report_file_already_downloaded(*args, **kwargs)
 342
 343     def report_unable_to_resume(self):
 344         """Report it was impossible to resume download."""
 345         self.to_screen('[download] Unable to resume')
 346
 347     @staticmethod
 348     def supports_manifest(manifest):
 349         """ Whether the downloader can download the fragments from the manifest.
 350         Redefine in subclasses if needed. """
 351         pass
 352
 353     def download(self, filename, info_dict, subtitle=False):
 354         """Download to a filename using the info from info_dict
 355         Return True on success and False otherwise
 356         """
 357
 358         nooverwrites_and_exists = (
 359             not self.params.get('overwrites', True)
 360             and os.path.exists(encodeFilename(filename))
 361         )
 362
 363         if not hasattr(filename, 'write'):
 364             continuedl_and_exists = (
 365                 self.params.get('continuedl', True)
 366                 and os.path.isfile(encodeFilename(filename))
 367                 and not self.params.get('nopart', False)
 368             )
 369
 370             # Check file already present
 371             if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
 372                 self.report_file_already_downloaded(filename)
 373                 self._hook_progress({
 374                     'filename': filename,
 375                     'status': 'finished',
 376                     'total_bytes': os.path.getsize(encodeFilename(filename)),
 377                 }, info_dict)
 378                 return True, False
 379
 380         if subtitle is False:
 381             min_sleep_interval = self.params.get('sleep_interval')
 382             if min_sleep_interval:
 383                 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
 384                 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
 385                 self.to_screen(
 386                     '[download] Sleeping %s seconds ...' % (
 387                         int(sleep_interval) if sleep_interval.is_integer()
 388                         else '%.2f' % sleep_interval))
 389                 time.sleep(sleep_interval)
 390         else:
 391             sleep_interval_sub = 0
 392             if type(self.params.get('sleep_interval_subtitles')) is int:
 393                 sleep_interval_sub = self.params.get('sleep_interval_subtitles')
 394             if sleep_interval_sub > 0:
 395                 self.to_screen(
 396                     '[download] Sleeping %s seconds ...' % (
 397                         sleep_interval_sub))
 398                 time.sleep(sleep_interval_sub)
 399         ret = self.real_download(filename, info_dict)
 400         self._finish_multiline_status()
 401         return ret, True
 402
 403     def real_download(self, filename, info_dict):
 404         """Real download process. Redefine in subclasses."""
 405         raise NotImplementedError('This method must be implemented by subclasses')
 406
 407     def _hook_progress(self, status, info_dict):
 408         if not self._progress_hooks:
 409             return
 410         status['info_dict'] = info_dict
 411         # youtube-dl passes the same status object to all the hooks.
 412         # Some third party scripts seems to be relying on this.
 413         # So keep this behavior if possible
 414         for ph in self._progress_hooks:
 415             ph(status)
 416
 417     def add_progress_hook(self, ph):
 418         # See YoutubeDl.py (search for progress_hooks) for a description of
 419         # this interface
 420         self._progress_hooks.append(ph)
 421
 422     def _debug_cmd(self, args, exe=None):
 423         if not self.params.get('verbose', False):
 424             return
 425
 426         str_args = [decodeArgument(a) for a in args]
 427
 428         if exe is None:
 429             exe = os.path.basename(str_args[0])
 430
 431         self.write_debug('%s command line: %s' % (exe, shell_quote(str_args)))