yt_dlp/downloader/hls.py

   1 from __future__ import unicode_literals
   2
   3 import re
   4 import binascii
   5 try:
   6     from Crypto.Cipher import AES
   7     can_decrypt_frag = True
   8 except ImportError:
   9     can_decrypt_frag = False
  10 try:
  11     import concurrent.futures
  12     can_threaded_download = True
  13 except ImportError:
  14     can_threaded_download = False
  15
  16 from ..downloader import _get_real_downloader
  17 from .fragment import FragmentFD
  18 from .external import FFmpegFD
  19
  20 from ..compat import (
  21     compat_urllib_error,
  22     compat_urlparse,
  23     compat_struct_pack,
  24 )
  25 from ..utils import (
  26     parse_m3u8_attributes,
  27     sanitize_open,
  28     update_url_query,
  29 )
  30
  31
  32 class HlsFD(FragmentFD):
  33     """
  34     Download segments in a m3u8 manifest. External downloaders can take over
  35     the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
  36     re-defining 'supports_manifest' function
  37     """
  38
  39     FD_NAME = 'hlsnative'
  40
  41     @staticmethod
  42     def can_download(manifest, info_dict, allow_unplayable_formats=False, with_crypto=can_decrypt_frag):
  43         UNSUPPORTED_FEATURES = [
  44             # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
  45
  46             # Live streams heuristic does not always work (e.g. geo restricted to Germany
  47             # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
  48             # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)',  # live streams [3]
  49
  50             # This heuristic also is not correct since segments may not be appended as well.
  51             # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
  52             # no segments will definitely be appended to the end of the playlist.
  53             # r'#EXT-X-PLAYLIST-TYPE:EVENT',  # media segments may be appended to the end of
  54             #                                 # event media playlists [4]
  55             # r'#EXT-X-MAP:',  # media initialization [5]
  56             # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
  57             # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
  58             # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
  59             # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
  60             # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
  61         ]
  62         if not allow_unplayable_formats:
  63             UNSUPPORTED_FEATURES += [
  64                 r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
  65             ]
  66
  67         def check_results():
  68             yield not info_dict.get('is_live')
  69             is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
  70             yield with_crypto or not is_aes128_enc
  71             yield not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest)
  72             for feature in UNSUPPORTED_FEATURES:
  73                 yield not re.search(feature, manifest)
  74         return all(check_results())
  75
  76     def real_download(self, filename, info_dict):
  77         man_url = info_dict['url']
  78         self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
  79
  80         urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
  81         man_url = urlh.geturl()
  82         s = urlh.read().decode('utf-8', 'ignore')
  83
  84         if not self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')):
  85             if info_dict.get('extra_param_to_segment_url') or info_dict.get('_decryption_key_url'):
  86                 self.report_error('pycryptodome not found. Please install')
  87                 return False
  88             if self.can_download(s, info_dict, with_crypto=True):
  89                 self.report_warning('pycryptodome is needed to download this file natively')
  90             fd = FFmpegFD(self.ydl, self.params)
  91             self.report_warning(
  92                 '%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename()))
  93             # TODO: Make progress updates work without hooking twice
  94             # for ph in self._progress_hooks:
  95             #     fd.add_progress_hook(ph)
  96             return fd.real_download(filename, info_dict)
  97
  98         real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
  99         if real_downloader and not real_downloader.supports_manifest(s):
 100             real_downloader = None
 101         if real_downloader:
 102             self.to_screen(
 103                 '[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
 104
 105         def is_ad_fragment_start(s):
 106             return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
 107                     or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad'))
 108
 109         def is_ad_fragment_end(s):
 110             return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s
 111                     or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment'))
 112
 113         fragments = []
 114
 115         media_frags = 0
 116         ad_frags = 0
 117         ad_frag_next = False
 118         for line in s.splitlines():
 119             line = line.strip()
 120             if not line:
 121                 continue
 122             if line.startswith('#'):
 123                 if is_ad_fragment_start(line):
 124                     ad_frag_next = True
 125                 elif is_ad_fragment_end(line):
 126                     ad_frag_next = False
 127                 continue
 128             if ad_frag_next:
 129                 ad_frags += 1
 130                 continue
 131             media_frags += 1
 132
 133         ctx = {
 134             'filename': filename,
 135             'total_frags': media_frags,
 136             'ad_frags': ad_frags,
 137         }
 138
 139         if real_downloader:
 140             self._prepare_external_frag_download(ctx)
 141         else:
 142             self._prepare_and_start_frag_download(ctx)
 143
 144         fragment_retries = self.params.get('fragment_retries', 0)
 145         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 146         test = self.params.get('test', False)
 147
 148         format_index = info_dict.get('format_index')
 149         extra_query = None
 150         extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
 151         if extra_param_to_segment_url:
 152             extra_query = compat_urlparse.parse_qs(extra_param_to_segment_url)
 153         i = 0
 154         media_sequence = 0
 155         decrypt_info = {'METHOD': 'NONE'}
 156         byte_range = {}
 157         discontinuity_count = 0
 158         frag_index = 0
 159         ad_frag_next = False
 160         for line in s.splitlines():
 161             line = line.strip()
 162             if line:
 163                 if not line.startswith('#'):
 164                     if format_index and discontinuity_count != format_index:
 165                         continue
 166                     if ad_frag_next:
 167                         continue
 168                     frag_index += 1
 169                     if frag_index <= ctx['fragment_index']:
 170                         continue
 171                     frag_url = (
 172                         line
 173                         if re.match(r'^https?://', line)
 174                         else compat_urlparse.urljoin(man_url, line))
 175                     if extra_query:
 176                         frag_url = update_url_query(frag_url, extra_query)
 177
 178                     fragments.append({
 179                         'frag_index': frag_index,
 180                         'url': frag_url,
 181                         'decrypt_info': decrypt_info,
 182                         'byte_range': byte_range,
 183                         'media_sequence': media_sequence,
 184                     })
 185
 186                 elif line.startswith('#EXT-X-MAP'):
 187                     if format_index and discontinuity_count != format_index:
 188                         continue
 189                     if frag_index > 0:
 190                         self.report_error(
 191                             'Initialization fragment found after media fragments, unable to download')
 192                         return False
 193                     frag_index += 1
 194                     map_info = parse_m3u8_attributes(line[11:])
 195                     frag_url = (
 196                         map_info.get('URI')
 197                         if re.match(r'^https?://', map_info.get('URI'))
 198                         else compat_urlparse.urljoin(man_url, map_info.get('URI')))
 199                     if extra_query:
 200                         frag_url = update_url_query(frag_url, extra_query)
 201
 202                     fragments.append({
 203                         'frag_index': frag_index,
 204                         'url': frag_url,
 205                         'decrypt_info': decrypt_info,
 206                         'byte_range': byte_range,
 207                         'media_sequence': media_sequence
 208                     })
 209
 210                     if map_info.get('BYTERANGE'):
 211                         splitted_byte_range = map_info.get('BYTERANGE').split('@')
 212                         sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
 213                         byte_range = {
 214                             'start': sub_range_start,
 215                             'end': sub_range_start + int(splitted_byte_range[0]),
 216                         }
 217
 218                 elif line.startswith('#EXT-X-KEY'):
 219                     decrypt_url = decrypt_info.get('URI')
 220                     decrypt_info = parse_m3u8_attributes(line[11:])
 221                     if decrypt_info['METHOD'] == 'AES-128':
 222                         if 'IV' in decrypt_info:
 223                             decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32))
 224                         if not re.match(r'^https?://', decrypt_info['URI']):
 225                             decrypt_info['URI'] = compat_urlparse.urljoin(
 226                                 man_url, decrypt_info['URI'])
 227                         if extra_query:
 228                             decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)
 229                         if decrypt_url != decrypt_info['URI']:
 230                             decrypt_info['KEY'] = None
 231
 232                 elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
 233                     media_sequence = int(line[22:])
 234                 elif line.startswith('#EXT-X-BYTERANGE'):
 235                     splitted_byte_range = line[17:].split('@')
 236                     sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end']
 237                     byte_range = {
 238                         'start': sub_range_start,
 239                         'end': sub_range_start + int(splitted_byte_range[0]),
 240                     }
 241                 elif is_ad_fragment_start(line):
 242                     ad_frag_next = True
 243                 elif is_ad_fragment_end(line):
 244                     ad_frag_next = False
 245                 elif line.startswith('#EXT-X-DISCONTINUITY'):
 246                     discontinuity_count += 1
 247                 i += 1
 248                 media_sequence += 1
 249
 250         # We only download the first fragment during the test
 251         if test:
 252             fragments = [fragments[0] if fragments else None]
 253
 254         if real_downloader:
 255             info_copy = info_dict.copy()
 256             info_copy['fragments'] = fragments
 257             fd = real_downloader(self.ydl, self.params)
 258             # TODO: Make progress updates work without hooking twice
 259             # for ph in self._progress_hooks:
 260             #     fd.add_progress_hook(ph)
 261             success = fd.real_download(filename, info_copy)
 262             if not success:
 263                 return False
 264         else:
 265             def download_fragment(fragment):
 266                 frag_index = fragment['frag_index']
 267                 frag_url = fragment['url']
 268                 decrypt_info = fragment['decrypt_info']
 269                 byte_range = fragment['byte_range']
 270                 media_sequence = fragment['media_sequence']
 271
 272                 ctx['fragment_index'] = frag_index
 273
 274                 count = 0
 275                 headers = info_dict.get('http_headers', {})
 276                 if byte_range:
 277                     headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
 278                 while count <= fragment_retries:
 279                     try:
 280                         success, frag_content = self._download_fragment(
 281                             ctx, frag_url, info_dict, headers)
 282                         if not success:
 283                             return False, frag_index
 284                         break
 285                     except compat_urllib_error.HTTPError as err:
 286                         # Unavailable (possibly temporary) fragments may be served.
 287                         # First we try to retry then either skip or abort.
 288                         # See https://github.com/ytdl-org/youtube-dl/issues/10165,
 289                         # https://github.com/ytdl-org/youtube-dl/issues/10448).
 290                         count += 1
 291                         if count <= fragment_retries:
 292                             self.report_retry_fragment(err, frag_index, count, fragment_retries)
 293                 if count > fragment_retries:
 294                     self.report_error('Giving up after %s fragment retries' % fragment_retries)
 295                     return False, frag_index
 296
 297                 if decrypt_info['METHOD'] == 'AES-128':
 298                     iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
 299                     decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(
 300                         self._prepare_url(info_dict, info_dict.get('_decryption_key_url') or decrypt_info['URI'])).read()
 301                     # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
 302                     # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
 303                     # not what it decrypts to.
 304                     if not test:
 305                         frag_content = AES.new(
 306                             decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
 307
 308                 return frag_content, frag_index
 309
 310             def append_fragment(frag_content, frag_index):
 311                 if frag_content:
 312                     fragment_filename = '%s-Frag%d' % (ctx['tmpfilename'], frag_index)
 313                     try:
 314                         file, frag_sanitized = sanitize_open(fragment_filename, 'rb')
 315                         ctx['fragment_filename_sanitized'] = frag_sanitized
 316                         file.close()
 317                         self._append_fragment(ctx, frag_content)
 318                         return True
 319                     except FileNotFoundError:
 320                         if skip_unavailable_fragments:
 321                             self.report_skip_fragment(frag_index)
 322                             return True
 323                         else:
 324                             self.report_error(
 325                                 'fragment %s not found, unable to continue' % frag_index)
 326                             return False
 327                 else:
 328                     if skip_unavailable_fragments:
 329                         self.report_skip_fragment(frag_index)
 330                         return True
 331                     else:
 332                         self.report_error(
 333                             'fragment %s not found, unable to continue' % frag_index)
 334                         return False
 335
 336             max_workers = self.params.get('concurrent_fragment_downloads', 1)
 337             if can_threaded_download and max_workers > 1:
 338                 self.report_warning('The download speed shown is only of one thread. This is a known issue')
 339                 with concurrent.futures.ThreadPoolExecutor(max_workers) as pool:
 340                     futures = [pool.submit(download_fragment, fragment) for fragment in fragments]
 341                     # timeout must be 0 to return instantly
 342                     done, not_done = concurrent.futures.wait(futures, timeout=0)
 343                     try:
 344                         while not_done:
 345                             # Check every 1 second for KeyboardInterrupt
 346                             freshly_done, not_done = concurrent.futures.wait(not_done, timeout=1)
 347                             done |= freshly_done
 348                     except KeyboardInterrupt:
 349                         for future in not_done:
 350                             future.cancel()
 351                         # timeout must be none to cancel
 352                         concurrent.futures.wait(not_done, timeout=None)
 353                         raise KeyboardInterrupt
 354                 results = [future.result() for future in futures]
 355
 356                 for frag_content, frag_index in results:
 357                     result = append_fragment(frag_content, frag_index)
 358                     if not result:
 359                         return False
 360             else:
 361                 for fragment in fragments:
 362                     frag_content, frag_index = download_fragment(fragment)
 363                     result = append_fragment(frag_content, frag_index)
 364                     if not result:
 365                         return False
 366
 367             self._finish_frag_download(ctx)
 368         return True