yt_dlp/downloader/common.py

   1 from __future__ import division, unicode_literals
   2
   3 import copy
   4 import os
   5 import re
   6 import sys
   7 import time
   8 import random
   9
  10 from ..compat import compat_os_name
  11 from ..utils import (
  12     decodeArgument,
  13     encodeFilename,
  14     error_to_compat_str,
  15     format_bytes,
  16     shell_quote,
  17     timeconvert,
  18 )
  19
  20
  21 class FileDownloader(object):
  22     """File Downloader class.
  23
  24     File downloader objects are the ones responsible of downloading the
  25     actual video file and writing it to disk.
  26
  27     File downloaders accept a lot of parameters. In order not to saturate
  28     the object constructor with arguments, it receives a dictionary of
  29     options instead.
  30
  31     Available options:
  32
  33     verbose:            Print additional info to stdout.
  34     quiet:              Do not print messages to stdout.
  35     ratelimit:          Download speed limit, in bytes/sec.
  36     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
  37     retries:            Number of times to retry for HTTP error 5xx
  38     buffersize:         Size of download buffer in bytes.
  39     noresizebuffer:     Do not automatically resize the download buffer.
  40     continuedl:         Try to continue downloads if possible.
  41     noprogress:         Do not print the progress bar.
  42     logtostderr:        Log messages to stderr instead of stdout.
  43     consoletitle:       Display progress in console window's titlebar.
  44     nopart:             Do not use temporary .part files.
  45     updatetime:         Use the Last-modified header to set output file timestamps.
  46     test:               Download only first bytes to test the downloader.
  47     min_filesize:       Skip files smaller than this size
  48     max_filesize:       Skip files larger than this size
  49     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
  50     external_downloader_args:  A dictionary of downloader keys (in lower case)
  51                         and a list of additional command-line arguments for the
  52                         executable. Use 'default' as the name for arguments to be
  53                         passed to all downloaders. For compatibility with youtube-dl,
  54                         a single list of args can also be used
  55     hls_use_mpegts:     Use the mpegts container for HLS videos.
  56     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
  57                         useful for bypassing bandwidth throttling imposed by
  58                         a webserver (experimental)
  59
  60     Subclasses of this one must re-define the real_download method.
  61     """
  62
  63     _TEST_FILE_SIZE = 10241
  64     params = None
  65
  66     def __init__(self, ydl, params):
  67         """Create a FileDownloader object with the given options."""
  68         self.ydl = ydl
  69         self._progress_hooks = []
  70         self.params = params
  71         self.add_progress_hook(self.report_progress)
  72
  73     @staticmethod
  74     def format_seconds(seconds):
  75         (mins, secs) = divmod(seconds, 60)
  76         (hours, mins) = divmod(mins, 60)
  77         if hours > 99:
  78             return '--:--:--'
  79         if hours == 0:
  80             return '%02d:%02d' % (mins, secs)
  81         else:
  82             return '%02d:%02d:%02d' % (hours, mins, secs)
  83
  84     @staticmethod
  85     def calc_percent(byte_counter, data_len):
  86         if data_len is None:
  87             return None
  88         return float(byte_counter) / float(data_len) * 100.0
  89
  90     @staticmethod
  91     def format_percent(percent):
  92         if percent is None:
  93             return '---.-%'
  94         return '%6s' % ('%3.1f%%' % percent)
  95
  96     @staticmethod
  97     def calc_eta(start, now, total, current):
  98         if total is None:
  99             return None
 100         if now is None:
 101             now = time.time()
 102         dif = now - start
 103         if current == 0 or dif < 0.001:  # One millisecond
 104             return None
 105         rate = float(current) / dif
 106         return int((float(total) - float(current)) / rate)
 107
 108     @staticmethod
 109     def format_eta(eta):
 110         if eta is None:
 111             return '--:--'
 112         return FileDownloader.format_seconds(eta)
 113
 114     @staticmethod
 115     def calc_speed(start, now, bytes):
 116         dif = now - start
 117         if bytes == 0 or dif < 0.001:  # One millisecond
 118             return None
 119         return float(bytes) / dif
 120
 121     @staticmethod
 122     def format_speed(speed):
 123         if speed is None:
 124             return '%10s' % '---b/s'
 125         return '%10s' % ('%s/s' % format_bytes(speed))
 126
 127     @staticmethod
 128     def format_retries(retries):
 129         return 'inf' if retries == float('inf') else '%.0f' % retries
 130
 131     @staticmethod
 132     def best_block_size(elapsed_time, bytes):
 133         new_min = max(bytes / 2.0, 1.0)
 134         new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
 135         if elapsed_time < 0.001:
 136             return int(new_max)
 137         rate = bytes / elapsed_time
 138         if rate > new_max:
 139             return int(new_max)
 140         if rate < new_min:
 141             return int(new_min)
 142         return int(rate)
 143
 144     @staticmethod
 145     def parse_bytes(bytestr):
 146         """Parse a string indicating a byte quantity into an integer."""
 147         matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 148         if matchobj is None:
 149             return None
 150         number = float(matchobj.group(1))
 151         multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 152         return int(round(number * multiplier))
 153
 154     def to_screen(self, *args, **kargs):
 155         self.ydl.to_stdout(*args, quiet=self.params.get('quiet'), **kargs)
 156
 157     def to_stderr(self, message):
 158         self.ydl.to_stderr(message)
 159
 160     def to_console_title(self, message):
 161         self.ydl.to_console_title(message)
 162
 163     def trouble(self, *args, **kargs):
 164         self.ydl.trouble(*args, **kargs)
 165
 166     def report_warning(self, *args, **kargs):
 167         self.ydl.report_warning(*args, **kargs)
 168
 169     def report_error(self, *args, **kargs):
 170         self.ydl.report_error(*args, **kargs)
 171
 172     def write_debug(self, *args, **kargs):
 173         self.ydl.write_debug(*args, **kargs)
 174
 175     def slow_down(self, start_time, now, byte_counter):
 176         """Sleep if the download speed is over the rate limit."""
 177         rate_limit = self.params.get('ratelimit')
 178         if rate_limit is None or byte_counter == 0:
 179             return
 180         if now is None:
 181             now = time.time()
 182         elapsed = now - start_time
 183         if elapsed <= 0.0:
 184             return
 185         speed = float(byte_counter) / elapsed
 186         if speed > rate_limit:
 187             sleep_time = float(byte_counter) / rate_limit - elapsed
 188             if sleep_time > 0:
 189                 time.sleep(sleep_time)
 190
 191     def temp_name(self, filename):
 192         """Returns a temporary filename for the given filename."""
 193         if self.params.get('nopart', False) or filename == '-' or \
 194                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 195             return filename
 196         return filename + '.part'
 197
 198     def undo_temp_name(self, filename):
 199         if filename.endswith('.part'):
 200             return filename[:-len('.part')]
 201         return filename
 202
 203     def ytdl_filename(self, filename):
 204         return filename + '.ytdl'
 205
 206     def try_rename(self, old_filename, new_filename):
 207         if old_filename == new_filename:
 208             return
 209         try:
 210             os.replace(old_filename, new_filename)
 211         except (IOError, OSError) as err:
 212             self.report_error(f'unable to rename file: {err}')
 213
 214     def try_utime(self, filename, last_modified_hdr):
 215         """Try to set the last-modified time of the given file."""
 216         if last_modified_hdr is None:
 217             return
 218         if not os.path.isfile(encodeFilename(filename)):
 219             return
 220         timestr = last_modified_hdr
 221         if timestr is None:
 222             return
 223         filetime = timeconvert(timestr)
 224         if filetime is None:
 225             return filetime
 226         # Ignore obviously invalid dates
 227         if filetime == 0:
 228             return
 229         try:
 230             os.utime(filename, (time.time(), filetime))
 231         except Exception:
 232             pass
 233         return filetime
 234
 235     def report_destination(self, filename):
 236         """Report destination filename."""
 237         self.to_screen('[download] Destination: ' + filename)
 238
 239     def _report_progress_status(self, msg, is_last_line=False):
 240         fullmsg = '[download] ' + msg
 241         if self.params.get('progress_with_newline', False):
 242             self.to_screen(fullmsg)
 243         else:
 244             if compat_os_name == 'nt':
 245                 prev_len = getattr(self, '_report_progress_prev_line_length',
 246                                    0)
 247                 if prev_len > len(fullmsg):
 248                     fullmsg += ' ' * (prev_len - len(fullmsg))
 249                 self._report_progress_prev_line_length = len(fullmsg)
 250                 clear_line = '\r'
 251             else:
 252                 clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
 253             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
 254         self.to_console_title('yt-dlp ' + msg)
 255
 256     def report_progress(self, s):
 257         if s['status'] == 'finished':
 258             if self.params.get('noprogress', False):
 259                 self.to_screen('[download] Download completed')
 260             else:
 261                 msg_template = '100%%'
 262                 if s.get('total_bytes') is not None:
 263                     s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 264                     msg_template += ' of %(_total_bytes_str)s'
 265                 if s.get('elapsed') is not None:
 266                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 267                     msg_template += ' in %(_elapsed_str)s'
 268                 self._report_progress_status(
 269                     msg_template % s, is_last_line=True)
 270
 271         if self.params.get('noprogress'):
 272             return
 273
 274         if s['status'] != 'downloading':
 275             return
 276
 277         if s.get('eta') is not None:
 278             s['_eta_str'] = self.format_eta(s['eta'])
 279         else:
 280             s['_eta_str'] = 'Unknown ETA'
 281
 282         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
 283             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
 284         elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
 285             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
 286         else:
 287             if s.get('downloaded_bytes') == 0:
 288                 s['_percent_str'] = self.format_percent(0)
 289             else:
 290                 s['_percent_str'] = 'Unknown %'
 291
 292         if s.get('speed') is not None:
 293             s['_speed_str'] = self.format_speed(s['speed'])
 294         else:
 295             s['_speed_str'] = 'Unknown speed'
 296
 297         if s.get('total_bytes') is not None:
 298             s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 299             msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
 300         elif s.get('total_bytes_estimate') is not None:
 301             s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
 302             msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
 303         else:
 304             if s.get('downloaded_bytes') is not None:
 305                 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
 306                 if s.get('elapsed'):
 307                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 308                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
 309                 else:
 310                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
 311             else:
 312                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
 313
 314         self._report_progress_status(msg_template % s)
 315
 316     def report_resuming_byte(self, resume_len):
 317         """Report attempt to resume at given byte."""
 318         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 319
 320     def report_retry(self, err, count, retries):
 321         """Report retry in case of HTTP error 5xx"""
 322         self.to_screen(
 323             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
 324             % (error_to_compat_str(err), count, self.format_retries(retries)))
 325
 326     def report_file_already_downloaded(self, *args, **kwargs):
 327         """Report file has already been fully downloaded."""
 328         return self.ydl.report_file_already_downloaded(*args, **kwargs)
 329
 330     def report_unable_to_resume(self):
 331         """Report it was impossible to resume download."""
 332         self.to_screen('[download] Unable to resume')
 333
 334     @staticmethod
 335     def supports_manifest(manifest):
 336         """ Whether the downloader can download the fragments from the manifest.
 337         Redefine in subclasses if needed. """
 338         pass
 339
 340     def download(self, filename, info_dict, subtitle=False):
 341         """Download to a filename using the info from info_dict
 342         Return True on success and False otherwise
 343         """
 344
 345         nooverwrites_and_exists = (
 346             not self.params.get('overwrites', True)
 347             and os.path.exists(encodeFilename(filename))
 348         )
 349
 350         if not hasattr(filename, 'write'):
 351             continuedl_and_exists = (
 352                 self.params.get('continuedl', True)
 353                 and os.path.isfile(encodeFilename(filename))
 354                 and not self.params.get('nopart', False)
 355             )
 356
 357             # Check file already present
 358             if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
 359                 self.report_file_already_downloaded(filename)
 360                 self._hook_progress({
 361                     'filename': filename,
 362                     'status': 'finished',
 363                     'total_bytes': os.path.getsize(encodeFilename(filename)),
 364                 }, info_dict)
 365                 return True, False
 366
 367         if subtitle is False:
 368             min_sleep_interval = self.params.get('sleep_interval')
 369             if min_sleep_interval:
 370                 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
 371                 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
 372                 self.to_screen(
 373                     '[download] Sleeping %s seconds ...' % (
 374                         int(sleep_interval) if sleep_interval.is_integer()
 375                         else '%.2f' % sleep_interval))
 376                 time.sleep(sleep_interval)
 377         else:
 378             sleep_interval_sub = 0
 379             if type(self.params.get('sleep_interval_subtitles')) is int:
 380                 sleep_interval_sub = self.params.get('sleep_interval_subtitles')
 381             if sleep_interval_sub > 0:
 382                 self.to_screen(
 383                     '[download] Sleeping %s seconds ...' % (
 384                         sleep_interval_sub))
 385                 time.sleep(sleep_interval_sub)
 386         return self.real_download(filename, info_dict), True
 387
 388     def real_download(self, filename, info_dict):
 389         """Real download process. Redefine in subclasses."""
 390         raise NotImplementedError('This method must be implemented by subclasses')
 391
 392     def _hook_progress(self, status, info_dict):
 393         if not self._progress_hooks:
 394             return
 395         info_dict = dict(info_dict)
 396         for key in ('__original_infodict', '__postprocessors'):
 397             info_dict.pop(key, None)
 398         # youtube-dl passes the same status object to all the hooks.
 399         # Some third party scripts seems to be relying on this.
 400         # So keep this behavior if possible
 401         status['info_dict'] = copy.deepcopy(info_dict)
 402         for ph in self._progress_hooks:
 403             ph(status)
 404
 405     def add_progress_hook(self, ph):
 406         # See YoutubeDl.py (search for progress_hooks) for a description of
 407         # this interface
 408         self._progress_hooks.append(ph)
 409
 410     def _debug_cmd(self, args, exe=None):
 411         if not self.params.get('verbose', False):
 412             return
 413
 414         str_args = [decodeArgument(a) for a in args]
 415
 416         if exe is None:
 417             exe = os.path.basename(str_args[0])
 418
 419         self.write_debug('%s command line: %s' % (exe, shell_quote(str_args)))