yt_dlp/downloader/common.py

   1 from __future__ import division, unicode_literals
   2
   3 import copy
   4 import os
   5 import re
   6 import sys
   7 import time
   8 import random
   9
  10 from ..compat import compat_os_name
  11 from ..utils import (
  12     decodeArgument,
  13     encodeFilename,
  14     error_to_compat_str,
  15     format_bytes,
  16     shell_quote,
  17     timeconvert,
  18 )
  19
  20
  21 class FileDownloader(object):
  22     """File Downloader class.
  23
  24     File downloader objects are the ones responsible of downloading the
  25     actual video file and writing it to disk.
  26
  27     File downloaders accept a lot of parameters. In order not to saturate
  28     the object constructor with arguments, it receives a dictionary of
  29     options instead.
  30
  31     Available options:
  32
  33     verbose:            Print additional info to stdout.
  34     quiet:              Do not print messages to stdout.
  35     ratelimit:          Download speed limit, in bytes/sec.
  36     throttledratelimit: Assume the download is being throttled below this speed (bytes/sec)
  37     retries:            Number of times to retry for HTTP error 5xx
  38     buffersize:         Size of download buffer in bytes.
  39     noresizebuffer:     Do not automatically resize the download buffer.
  40     continuedl:         Try to continue downloads if possible.
  41     noprogress:         Do not print the progress bar.
  42     logtostderr:        Log messages to stderr instead of stdout.
  43     consoletitle:       Display progress in console window's titlebar.
  44     nopart:             Do not use temporary .part files.
  45     updatetime:         Use the Last-modified header to set output file timestamps.
  46     test:               Download only first bytes to test the downloader.
  47     min_filesize:       Skip files smaller than this size
  48     max_filesize:       Skip files larger than this size
  49     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
  50     external_downloader_args:  A list of additional command-line arguments for the
  51                         external downloader.
  52     hls_use_mpegts:     Use the mpegts container for HLS videos.
  53     http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
  54                         useful for bypassing bandwidth throttling imposed by
  55                         a webserver (experimental)
  56
  57     Subclasses of this one must re-define the real_download method.
  58     """
  59
  60     _TEST_FILE_SIZE = 10241
  61     params = None
  62
  63     def __init__(self, ydl, params):
  64         """Create a FileDownloader object with the given options."""
  65         self.ydl = ydl
  66         self._progress_hooks = []
  67         self.params = params
  68         self.add_progress_hook(self.report_progress)
  69
  70     @staticmethod
  71     def format_seconds(seconds):
  72         (mins, secs) = divmod(seconds, 60)
  73         (hours, mins) = divmod(mins, 60)
  74         if hours > 99:
  75             return '--:--:--'
  76         if hours == 0:
  77             return '%02d:%02d' % (mins, secs)
  78         else:
  79             return '%02d:%02d:%02d' % (hours, mins, secs)
  80
  81     @staticmethod
  82     def calc_percent(byte_counter, data_len):
  83         if data_len is None:
  84             return None
  85         return float(byte_counter) / float(data_len) * 100.0
  86
  87     @staticmethod
  88     def format_percent(percent):
  89         if percent is None:
  90             return '---.-%'
  91         return '%6s' % ('%3.1f%%' % percent)
  92
  93     @staticmethod
  94     def calc_eta(start, now, total, current):
  95         if total is None:
  96             return None
  97         if now is None:
  98             now = time.time()
  99         dif = now - start
 100         if current == 0 or dif < 0.001:  # One millisecond
 101             return None
 102         rate = float(current) / dif
 103         return int((float(total) - float(current)) / rate)
 104
 105     @staticmethod
 106     def format_eta(eta):
 107         if eta is None:
 108             return '--:--'
 109         return FileDownloader.format_seconds(eta)
 110
 111     @staticmethod
 112     def calc_speed(start, now, bytes):
 113         dif = now - start
 114         if bytes == 0 or dif < 0.001:  # One millisecond
 115             return None
 116         return float(bytes) / dif
 117
 118     @staticmethod
 119     def format_speed(speed):
 120         if speed is None:
 121             return '%10s' % '---b/s'
 122         return '%10s' % ('%s/s' % format_bytes(speed))
 123
 124     @staticmethod
 125     def format_retries(retries):
 126         return 'inf' if retries == float('inf') else '%.0f' % retries
 127
 128     @staticmethod
 129     def best_block_size(elapsed_time, bytes):
 130         new_min = max(bytes / 2.0, 1.0)
 131         new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
 132         if elapsed_time < 0.001:
 133             return int(new_max)
 134         rate = bytes / elapsed_time
 135         if rate > new_max:
 136             return int(new_max)
 137         if rate < new_min:
 138             return int(new_min)
 139         return int(rate)
 140
 141     @staticmethod
 142     def parse_bytes(bytestr):
 143         """Parse a string indicating a byte quantity into an integer."""
 144         matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 145         if matchobj is None:
 146             return None
 147         number = float(matchobj.group(1))
 148         multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 149         return int(round(number * multiplier))
 150
 151     def to_screen(self, *args, **kargs):
 152         self.ydl.to_stdout(*args, quiet=self.params.get('quiet'), **kargs)
 153
 154     def to_stderr(self, message):
 155         self.ydl.to_stderr(message)
 156
 157     def to_console_title(self, message):
 158         self.ydl.to_console_title(message)
 159
 160     def trouble(self, *args, **kargs):
 161         self.ydl.trouble(*args, **kargs)
 162
 163     def report_warning(self, *args, **kargs):
 164         self.ydl.report_warning(*args, **kargs)
 165
 166     def report_error(self, *args, **kargs):
 167         self.ydl.report_error(*args, **kargs)
 168
 169     def write_debug(self, *args, **kargs):
 170         self.ydl.write_debug(*args, **kargs)
 171
 172     def slow_down(self, start_time, now, byte_counter):
 173         """Sleep if the download speed is over the rate limit."""
 174         rate_limit = self.params.get('ratelimit')
 175         if rate_limit is None or byte_counter == 0:
 176             return
 177         if now is None:
 178             now = time.time()
 179         elapsed = now - start_time
 180         if elapsed <= 0.0:
 181             return
 182         speed = float(byte_counter) / elapsed
 183         if speed > rate_limit:
 184             sleep_time = float(byte_counter) / rate_limit - elapsed
 185             if sleep_time > 0:
 186                 time.sleep(sleep_time)
 187
 188     def temp_name(self, filename):
 189         """Returns a temporary filename for the given filename."""
 190         if self.params.get('nopart', False) or filename == '-' or \
 191                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 192             return filename
 193         return filename + '.part'
 194
 195     def undo_temp_name(self, filename):
 196         if filename.endswith('.part'):
 197             return filename[:-len('.part')]
 198         return filename
 199
 200     def ytdl_filename(self, filename):
 201         return filename + '.ytdl'
 202
 203     def try_rename(self, old_filename, new_filename):
 204         try:
 205             if old_filename == new_filename:
 206                 return
 207             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 208         except (IOError, OSError) as err:
 209             self.report_error('unable to rename file: %s' % error_to_compat_str(err))
 210
 211     def try_utime(self, filename, last_modified_hdr):
 212         """Try to set the last-modified time of the given file."""
 213         if last_modified_hdr is None:
 214             return
 215         if not os.path.isfile(encodeFilename(filename)):
 216             return
 217         timestr = last_modified_hdr
 218         if timestr is None:
 219             return
 220         filetime = timeconvert(timestr)
 221         if filetime is None:
 222             return filetime
 223         # Ignore obviously invalid dates
 224         if filetime == 0:
 225             return
 226         try:
 227             os.utime(filename, (time.time(), filetime))
 228         except Exception:
 229             pass
 230         return filetime
 231
 232     def report_destination(self, filename):
 233         """Report destination filename."""
 234         self.to_screen('[download] Destination: ' + filename)
 235
 236     def _report_progress_status(self, msg, is_last_line=False):
 237         fullmsg = '[download] ' + msg
 238         if self.params.get('progress_with_newline', False):
 239             self.to_screen(fullmsg)
 240         else:
 241             if compat_os_name == 'nt':
 242                 prev_len = getattr(self, '_report_progress_prev_line_length',
 243                                    0)
 244                 if prev_len > len(fullmsg):
 245                     fullmsg += ' ' * (prev_len - len(fullmsg))
 246                 self._report_progress_prev_line_length = len(fullmsg)
 247                 clear_line = '\r'
 248             else:
 249                 clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
 250             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
 251         self.to_console_title('yt-dlp ' + msg)
 252
 253     def report_progress(self, s):
 254         if s['status'] == 'finished':
 255             if self.params.get('noprogress', False):
 256                 self.to_screen('[download] Download completed')
 257             else:
 258                 msg_template = '100%%'
 259                 if s.get('total_bytes') is not None:
 260                     s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 261                     msg_template += ' of %(_total_bytes_str)s'
 262                 if s.get('elapsed') is not None:
 263                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 264                     msg_template += ' in %(_elapsed_str)s'
 265                 self._report_progress_status(
 266                     msg_template % s, is_last_line=True)
 267
 268         if self.params.get('noprogress'):
 269             return
 270
 271         if s['status'] != 'downloading':
 272             return
 273
 274         if s.get('eta') is not None:
 275             s['_eta_str'] = self.format_eta(s['eta'])
 276         else:
 277             s['_eta_str'] = 'Unknown ETA'
 278
 279         if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
 280             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
 281         elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
 282             s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
 283         else:
 284             if s.get('downloaded_bytes') == 0:
 285                 s['_percent_str'] = self.format_percent(0)
 286             else:
 287                 s['_percent_str'] = 'Unknown %'
 288
 289         if s.get('speed') is not None:
 290             s['_speed_str'] = self.format_speed(s['speed'])
 291         else:
 292             s['_speed_str'] = 'Unknown speed'
 293
 294         if s.get('total_bytes') is not None:
 295             s['_total_bytes_str'] = format_bytes(s['total_bytes'])
 296             msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
 297         elif s.get('total_bytes_estimate') is not None:
 298             s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
 299             msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
 300         else:
 301             if s.get('downloaded_bytes') is not None:
 302                 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
 303                 if s.get('elapsed'):
 304                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
 305                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
 306                 else:
 307                     msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
 308             else:
 309                 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
 310
 311         self._report_progress_status(msg_template % s)
 312
 313     def report_resuming_byte(self, resume_len):
 314         """Report attempt to resume at given byte."""
 315         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 316
 317     def report_retry(self, err, count, retries):
 318         """Report retry in case of HTTP error 5xx"""
 319         self.to_screen(
 320             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
 321             % (error_to_compat_str(err), count, self.format_retries(retries)))
 322
 323     def report_file_already_downloaded(self, *args, **kwargs):
 324         """Report file has already been fully downloaded."""
 325         return self.ydl.report_file_already_downloaded(*args, **kwargs)
 326
 327     def report_unable_to_resume(self):
 328         """Report it was impossible to resume download."""
 329         self.to_screen('[download] Unable to resume')
 330
 331     @staticmethod
 332     def supports_manifest(manifest):
 333         """ Whether the downloader can download the fragments from the manifest.
 334         Redefine in subclasses if needed. """
 335         pass
 336
 337     def download(self, filename, info_dict, subtitle=False):
 338         """Download to a filename using the info from info_dict
 339         Return True on success and False otherwise
 340         """
 341
 342         nooverwrites_and_exists = (
 343             not self.params.get('overwrites', True)
 344             and os.path.exists(encodeFilename(filename))
 345         )
 346
 347         if not hasattr(filename, 'write'):
 348             continuedl_and_exists = (
 349                 self.params.get('continuedl', True)
 350                 and os.path.isfile(encodeFilename(filename))
 351                 and not self.params.get('nopart', False)
 352             )
 353
 354             # Check file already present
 355             if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
 356                 self.report_file_already_downloaded(filename)
 357                 self._hook_progress({
 358                     'filename': filename,
 359                     'status': 'finished',
 360                     'total_bytes': os.path.getsize(encodeFilename(filename)),
 361                 }, info_dict)
 362                 return True, False
 363
 364         if subtitle is False:
 365             min_sleep_interval = self.params.get('sleep_interval')
 366             if min_sleep_interval:
 367                 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
 368                 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
 369                 self.to_screen(
 370                     '[download] Sleeping %s seconds ...' % (
 371                         int(sleep_interval) if sleep_interval.is_integer()
 372                         else '%.2f' % sleep_interval))
 373                 time.sleep(sleep_interval)
 374         else:
 375             sleep_interval_sub = 0
 376             if type(self.params.get('sleep_interval_subtitles')) is int:
 377                 sleep_interval_sub = self.params.get('sleep_interval_subtitles')
 378             if sleep_interval_sub > 0:
 379                 self.to_screen(
 380                     '[download] Sleeping %s seconds ...' % (
 381                         sleep_interval_sub))
 382                 time.sleep(sleep_interval_sub)
 383         return self.real_download(filename, info_dict), True
 384
 385     def real_download(self, filename, info_dict):
 386         """Real download process. Redefine in subclasses."""
 387         raise NotImplementedError('This method must be implemented by subclasses')
 388
 389     def _hook_progress(self, status, info_dict):
 390         if not self._progress_hooks:
 391             return
 392         info_dict = dict(info_dict)
 393         for key in ('__original_infodict', '__postprocessors'):
 394             info_dict.pop(key, None)
 395         # youtube-dl passes the same status object to all the hooks.
 396         # Some third party scripts seems to be relying on this.
 397         # So keep this behavior if possible
 398         status['info_dict'] = copy.deepcopy(info_dict)
 399         for ph in self._progress_hooks:
 400             ph(status)
 401
 402     def add_progress_hook(self, ph):
 403         # See YoutubeDl.py (search for progress_hooks) for a description of
 404         # this interface
 405         self._progress_hooks.append(ph)
 406
 407     def _debug_cmd(self, args, exe=None):
 408         if not self.params.get('verbose', False):
 409             return
 410
 411         str_args = [decodeArgument(a) for a in args]
 412
 413         if exe is None:
 414             exe = os.path.basename(str_args[0])
 415
 416         self.write_debug('%s command line: %s' % (exe, shell_quote(str_args)))