yt_dlp/postprocessor/common.py

   1 from __future__ import unicode_literals
   2
   3 import functools
   4 import itertools
   5 import json
   6 import os
   7 import time
   8 import urllib.error
   9
  10 from ..utils import (
  11     _configuration_args,
  12     encodeFilename,
  13     network_exceptions,
  14     PostProcessingError,
  15     sanitized_Request,
  16     write_string,
  17 )
  18
  19
  20 class PostProcessorMetaClass(type):
  21     @staticmethod
  22     def run_wrapper(func):
  23         @functools.wraps(func)
  24         def run(self, info, *args, **kwargs):
  25             info_copy = self._copy_infodict(info)
  26             self._hook_progress({'status': 'started'}, info_copy)
  27             ret = func(self, info, *args, **kwargs)
  28             if ret is not None:
  29                 _, info = ret
  30             self._hook_progress({'status': 'finished'}, info_copy)
  31             return ret
  32         return run
  33
  34     def __new__(cls, name, bases, attrs):
  35         if 'run' in attrs:
  36             attrs['run'] = cls.run_wrapper(attrs['run'])
  37         return type.__new__(cls, name, bases, attrs)
  38
  39
  40 class PostProcessor(metaclass=PostProcessorMetaClass):
  41     """Post Processor class.
  42
  43     PostProcessor objects can be added to downloaders with their
  44     add_post_processor() method. When the downloader has finished a
  45     successful download, it will take its internal chain of PostProcessors
  46     and start calling the run() method on each one of them, first with
  47     an initial argument and then with the returned value of the previous
  48     PostProcessor.
  49
  50     The chain will be stopped if one of them ever returns None or the end
  51     of the chain is reached.
  52
  53     PostProcessor objects follow a "mutual registration" process similar
  54     to InfoExtractor objects.
  55
  56     Optionally PostProcessor can use a list of additional command-line arguments
  57     with self._configuration_args.
  58     """
  59
  60     _downloader = None
  61
  62     def __init__(self, downloader=None):
  63         self._progress_hooks = []
  64         self.add_progress_hook(self.report_progress)
  65         self.set_downloader(downloader)
  66         self.PP_NAME = self.pp_key()
  67
  68     @classmethod
  69     def pp_key(cls):
  70         name = cls.__name__[:-2]
  71         return name[6:] if name[:6].lower() == 'ffmpeg' else name
  72
  73     def to_screen(self, text, prefix=True, *args, **kwargs):
  74         tag = '[%s] ' % self.PP_NAME if prefix else ''
  75         if self._downloader:
  76             return self._downloader.to_screen('%s%s' % (tag, text), *args, **kwargs)
  77
  78     def report_warning(self, text, *args, **kwargs):
  79         if self._downloader:
  80             return self._downloader.report_warning(text, *args, **kwargs)
  81
  82     def deprecation_warning(self, text):
  83         if self._downloader:
  84             return self._downloader.deprecation_warning(text)
  85         write_string(f'DeprecationWarning: {text}')
  86
  87     def report_error(self, text, *args, **kwargs):
  88         # Exists only for compatibility. Do not use
  89         if self._downloader:
  90             return self._downloader.report_error(text, *args, **kwargs)
  91
  92     def write_debug(self, text, *args, **kwargs):
  93         if self._downloader:
  94             return self._downloader.write_debug(text, *args, **kwargs)
  95
  96     def get_param(self, name, default=None, *args, **kwargs):
  97         if self._downloader:
  98             return self._downloader.params.get(name, default, *args, **kwargs)
  99         return default
 100
 101     def set_downloader(self, downloader):
 102         """Sets the downloader for this PP."""
 103         self._downloader = downloader
 104         for ph in getattr(downloader, '_postprocessor_hooks', []):
 105             self.add_progress_hook(ph)
 106
 107     def _copy_infodict(self, info_dict):
 108         return getattr(self._downloader, '_copy_infodict', dict)(info_dict)
 109
 110     @staticmethod
 111     def _restrict_to(*, video=True, audio=True, images=True, simulated=True):
 112         allowed = {'video': video, 'audio': audio, 'images': images}
 113
 114         def decorator(func):
 115             @functools.wraps(func)
 116             def wrapper(self, info):
 117                 if not simulated and (self.get_param('simulate') or self.get_param('skip_download')):
 118                     return [], info
 119                 format_type = (
 120                     'video' if info.get('vcodec') != 'none'
 121                     else 'audio' if info.get('acodec') != 'none'
 122                     else 'images')
 123                 if allowed[format_type]:
 124                     return func(self, info)
 125                 else:
 126                     self.to_screen('Skipping %s' % format_type)
 127                     return [], info
 128             return wrapper
 129         return decorator
 130
 131     def run(self, information):
 132         """Run the PostProcessor.
 133
 134         The "information" argument is a dictionary like the ones
 135         composed by InfoExtractors. The only difference is that this
 136         one has an extra field called "filepath" that points to the
 137         downloaded file.
 138
 139         This method returns a tuple, the first element is a list of the files
 140         that can be deleted, and the second of which is the updated
 141         information.
 142
 143         In addition, this method may raise a PostProcessingError
 144         exception if post processing fails.
 145         """
 146         return [], information  # by default, keep file and do nothing
 147
 148     def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'):
 149         try:
 150             os.utime(encodeFilename(path), (atime, mtime))
 151         except Exception:
 152             self.report_warning(errnote)
 153
 154     def _configuration_args(self, exe, *args, **kwargs):
 155         return _configuration_args(
 156             self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs)
 157
 158     def _hook_progress(self, status, info_dict):
 159         if not self._progress_hooks:
 160             return
 161         status.update({
 162             'info_dict': info_dict,
 163             'postprocessor': self.pp_key(),
 164         })
 165         for ph in self._progress_hooks:
 166             ph(status)
 167
 168     def add_progress_hook(self, ph):
 169         # See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface
 170         self._progress_hooks.append(ph)
 171
 172     def report_progress(self, s):
 173         s['_default_template'] = '%(postprocessor)s %(status)s' % s
 174
 175         progress_dict = s.copy()
 176         progress_dict.pop('info_dict')
 177         progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
 178
 179         progress_template = self.get_param('progress_template', {})
 180         tmpl = progress_template.get('postprocess')
 181         if tmpl:
 182             self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict))
 183
 184         self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
 185             progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s',
 186             progress_dict))
 187
 188     def _download_json(self, url, *, expected_http_errors=(404,)):
 189         # While this is not an extractor, it behaves similar to one and
 190         # so obey extractor_retries and sleep_interval_requests
 191         max_retries = self.get_param('extractor_retries', 3)
 192         sleep_interval = self.get_param('sleep_interval_requests') or 0
 193
 194         self.write_debug(f'{self.PP_NAME} query: {url}')
 195         for retries in itertools.count():
 196             try:
 197                 rsp = self._downloader.urlopen(sanitized_Request(url))
 198                 return json.loads(rsp.read().decode(rsp.info().get_param('charset') or 'utf-8'))
 199             except network_exceptions as e:
 200                 if isinstance(e, urllib.error.HTTPError) and e.code in expected_http_errors:
 201                     return None
 202                 if retries < max_retries:
 203                     self.report_warning(f'{e}. Retrying...')
 204                     if sleep_interval > 0:
 205                         self.to_screen(f'Sleeping {sleep_interval} seconds ...')
 206                         time.sleep(sleep_interval)
 207                     continue
 208                 raise PostProcessingError(f'Unable to communicate with {self.PP_NAME} API: {e}')
 209
 210
 211 class AudioConversionError(PostProcessingError):
 212     pass