1 from __future__
import unicode_literals
8 from Crypto
.Cipher
import AES
9 can_decrypt_frag
= True
11 can_decrypt_frag
= False
13 import concurrent
.futures
14 can_threaded_download
= True
16 can_threaded_download
= False
18 from ..downloader
import _get_real_downloader
19 from .fragment
import FragmentFD
20 from .external
import FFmpegFD
22 from ..compat
import (
28 parse_m3u8_attributes
,
36 class HlsFD(FragmentFD
):
38 Download segments in a m3u8 manifest. External downloaders can take over
39 the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
40 re-defining 'supports_manifest' function
46 def can_download(manifest
, info_dict
, allow_unplayable_formats
=False, with_crypto
=can_decrypt_frag
):
47 UNSUPPORTED_FEATURES
= [
48 # r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2]
50 # Live streams heuristic does not always work (e.g. geo restricted to Germany
51 # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
52 # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', # live streams [3]
54 # This heuristic also is not correct since segments may not be appended as well.
55 # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
56 # no segments will definitely be appended to the end of the playlist.
57 # r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of
58 # # event media playlists [4]
59 # r'#EXT-X-MAP:', # media initialization [5]
60 # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
61 # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
62 # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
63 # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
64 # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
66 if not allow_unplayable_formats
:
67 UNSUPPORTED_FEATURES
+= [
68 r
'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1]
72 yield not info_dict
.get('is_live')
73 is_aes128_enc
= '#EXT-X-KEY:METHOD=AES-128' in manifest
74 yield with_crypto
or not is_aes128_enc
75 yield not (is_aes128_enc
and r
'#EXT-X-BYTERANGE' in manifest
)
76 for feature
in UNSUPPORTED_FEATURES
:
77 yield not re
.search(feature
, manifest
)
78 return all(check_results())
80 def real_download(self
, filename
, info_dict
):
81 man_url
= info_dict
['url']
82 self
.to_screen('[%s] Downloading m3u8 manifest' % self
.FD_NAME
)
84 is_webvtt
= info_dict
['ext'] == 'vtt'
86 urlh
= self
.ydl
.urlopen(self
._prepare
_url
(info_dict
, man_url
))
87 man_url
= urlh
.geturl()
88 s
= urlh
.read().decode('utf-8', 'ignore')
90 if not self
.can_download(s
, info_dict
, self
.params
.get('allow_unplayable_formats')):
91 if info_dict
.get('extra_param_to_segment_url') or info_dict
.get('_decryption_key_url'):
92 self
.report_error('pycryptodome not found. Please install')
94 if self
.can_download(s
, info_dict
, with_crypto
=True):
95 self
.report_warning('pycryptodome is needed to download this file natively')
96 fd
= FFmpegFD(self
.ydl
, self
.params
)
98 '%s detected unsupported features; extraction will be delegated to %s' % (self
.FD_NAME
, fd
.get_basename()))
99 # TODO: Make progress updates work without hooking twice
100 # for ph in self._progress_hooks:
101 # fd.add_progress_hook(ph)
102 return fd
.real_download(filename
, info_dict
)
104 real_downloader
= _get_real_downloader(info_dict
, 'm3u8_frag_urls', self
.params
, None)
105 if real_downloader
and not real_downloader
.supports_manifest(s
):
106 real_downloader
= None
109 '[%s] Fragment downloads will be delegated to %s' % (self
.FD_NAME
, real_downloader
.get_basename()))
111 def is_ad_fragment_start(s
):
112 return (s
.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
113 or s
.startswith('#UPLYNK-SEGMENT') and s
.endswith(',ad'))
115 def is_ad_fragment_end(s
):
116 return (s
.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s
117 or s
.startswith('#UPLYNK-SEGMENT') and s
.endswith(',segment'))
124 for line
in s
.splitlines():
128 if line
.startswith('#'):
129 if is_ad_fragment_start(line
):
131 elif is_ad_fragment_end(line
):
140 'filename': filename
,
141 'total_frags': media_frags
,
142 'ad_frags': ad_frags
,
146 self
._prepare
_external
_frag
_download
(ctx
)
148 self
._prepare
_and
_start
_frag
_download
(ctx
)
150 extra_state
= ctx
.setdefault('extra_state', {})
152 fragment_retries
= self
.params
.get('fragment_retries', 0)
153 skip_unavailable_fragments
= self
.params
.get('skip_unavailable_fragments', True)
154 test
= self
.params
.get('test', False)
156 format_index
= info_dict
.get('format_index')
158 extra_param_to_segment_url
= info_dict
.get('extra_param_to_segment_url')
159 if extra_param_to_segment_url
:
160 extra_query
= compat_urlparse
.parse_qs(extra_param_to_segment_url
)
163 decrypt_info
= {'METHOD': 'NONE'}
165 discontinuity_count
= 0
168 for line
in s
.splitlines():
171 if not line
.startswith('#'):
172 if format_index
and discontinuity_count
!= format_index
:
177 if frag_index
<= ctx
['fragment_index']:
181 if re
.match(r
'^https?://', line
)
182 else compat_urlparse
.urljoin(man_url
, line
))
184 frag_url
= update_url_query(frag_url
, extra_query
)
187 'frag_index': frag_index
,
189 'decrypt_info': decrypt_info
,
190 'byte_range': byte_range
,
191 'media_sequence': media_sequence
,
194 elif line
.startswith('#EXT-X-MAP'):
195 if format_index
and discontinuity_count
!= format_index
:
199 'Initialization fragment found after media fragments, unable to download')
202 map_info
= parse_m3u8_attributes(line
[11:])
205 if re
.match(r
'^https?://', map_info
.get('URI'))
206 else compat_urlparse
.urljoin(man_url
, map_info
.get('URI')))
208 frag_url
= update_url_query(frag_url
, extra_query
)
211 'frag_index': frag_index
,
213 'decrypt_info': decrypt_info
,
214 'byte_range': byte_range
,
215 'media_sequence': media_sequence
218 if map_info
.get('BYTERANGE'):
219 splitted_byte_range
= map_info
.get('BYTERANGE').split('@')
220 sub_range_start
= int(splitted_byte_range
[1]) if len(splitted_byte_range
) == 2 else byte_range
['end']
222 'start': sub_range_start
,
223 'end': sub_range_start
+ int(splitted_byte_range
[0]),
226 elif line
.startswith('#EXT-X-KEY'):
227 decrypt_url
= decrypt_info
.get('URI')
228 decrypt_info
= parse_m3u8_attributes(line
[11:])
229 if decrypt_info
['METHOD'] == 'AES-128':
230 if 'IV' in decrypt_info
:
231 decrypt_info
['IV'] = binascii
.unhexlify(decrypt_info
['IV'][2:].zfill(32))
232 if not re
.match(r
'^https?://', decrypt_info
['URI']):
233 decrypt_info
['URI'] = compat_urlparse
.urljoin(
234 man_url
, decrypt_info
['URI'])
236 decrypt_info
['URI'] = update_url_query(decrypt_info
['URI'], extra_query
)
237 if decrypt_url
!= decrypt_info
['URI']:
238 decrypt_info
['KEY'] = None
240 elif line
.startswith('#EXT-X-MEDIA-SEQUENCE'):
241 media_sequence
= int(line
[22:])
242 elif line
.startswith('#EXT-X-BYTERANGE'):
243 splitted_byte_range
= line
[17:].split('@')
244 sub_range_start
= int(splitted_byte_range
[1]) if len(splitted_byte_range
) == 2 else byte_range
['end']
246 'start': sub_range_start
,
247 'end': sub_range_start
+ int(splitted_byte_range
[0]),
249 elif is_ad_fragment_start(line
):
251 elif is_ad_fragment_end(line
):
253 elif line
.startswith('#EXT-X-DISCONTINUITY'):
254 discontinuity_count
+= 1
258 # We only download the first fragment during the test
260 fragments
= [fragments
[0] if fragments
else None]
263 info_copy
= info_dict
.copy()
264 info_copy
['fragments'] = fragments
265 fd
= real_downloader(self
.ydl
, self
.params
)
266 # TODO: Make progress updates work without hooking twice
267 # for ph in self._progress_hooks:
268 # fd.add_progress_hook(ph)
269 success
= fd
.real_download(filename
, info_copy
)
273 def download_fragment(fragment
):
274 frag_index
= fragment
['frag_index']
275 frag_url
= fragment
['url']
276 decrypt_info
= fragment
['decrypt_info']
277 byte_range
= fragment
['byte_range']
278 media_sequence
= fragment
['media_sequence']
280 ctx
['fragment_index'] = frag_index
283 headers
= info_dict
.get('http_headers', {})
285 headers
['Range'] = 'bytes=%d-%d' % (byte_range
['start'], byte_range
['end'] - 1)
286 while count
<= fragment_retries
:
288 success
, frag_content
= self
._download
_fragment
(
289 ctx
, frag_url
, info_dict
, headers
)
291 return False, frag_index
293 except compat_urllib_error
.HTTPError
as err
:
294 # Unavailable (possibly temporary) fragments may be served.
295 # First we try to retry then either skip or abort.
296 # See https://github.com/ytdl-org/youtube-dl/issues/10165,
297 # https://github.com/ytdl-org/youtube-dl/issues/10448).
299 if count
<= fragment_retries
:
300 self
.report_retry_fragment(err
, frag_index
, count
, fragment_retries
)
301 if count
> fragment_retries
:
302 ctx
['dest_stream'].close()
303 self
.report_error('Giving up after %s fragment retries' % fragment_retries
)
304 return False, frag_index
306 if decrypt_info
['METHOD'] == 'AES-128':
307 iv
= decrypt_info
.get('IV') or compat_struct_pack('>8xq', media_sequence
)
308 decrypt_info
['KEY'] = decrypt_info
.get('KEY') or self
.ydl
.urlopen(
309 self
._prepare
_url
(info_dict
, info_dict
.get('_decryption_key_url') or decrypt_info
['URI'])).read()
310 # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
311 # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
312 # not what it decrypts to.
314 frag_content
= AES
.new(
315 decrypt_info
['KEY'], AES
.MODE_CBC
, iv
).decrypt(frag_content
)
317 return frag_content
, frag_index
319 pack_fragment
= lambda frag_content
, _
: frag_content
322 def pack_fragment(frag_content
, frag_index
):
323 output
= io
.StringIO()
325 for block
in webvtt
.parse_fragment(frag_content
):
326 if isinstance(block
, webvtt
.CueBlock
):
327 block
.start
+= adjust
330 dedup_window
= extra_state
.setdefault('webvtt_dedup_window', [])
333 # skip the cue if an identical one appears
334 # in the window of potential duplicates
335 # and prune the window of unviable candidates
338 while i
< len(dedup_window
):
339 window_cue
= dedup_window
[i
]
340 if window_cue
== cue
:
342 if window_cue
['end'] >= cue
['start']:
352 # add the cue to the window
353 dedup_window
.append(cue
)
354 elif isinstance(block
, webvtt
.Magic
):
355 # take care of MPEG PES timestamp overflow
356 if block
.mpegts
is None:
358 extra_state
.setdefault('webvtt_mpegts_adjust', 0)
359 block
.mpegts
+= extra_state
['webvtt_mpegts_adjust'] << 33
360 if block
.mpegts
< extra_state
.get('webvtt_mpegts_last', 0):
361 extra_state
['webvtt_mpegts_adjust'] += 1
362 block
.mpegts
+= 1 << 33
363 extra_state
['webvtt_mpegts_last'] = block
.mpegts
366 extra_state
['webvtt_mpegts'] = block
.mpegts
or 0
367 extra_state
['webvtt_local'] = block
.local
or 0
368 # XXX: block.local = block.mpegts = None ?
370 if block
.mpegts
is not None and block
.local
is not None:
372 (block
.mpegts
- extra_state
.get('webvtt_mpegts', 0))
373 - (block
.local
- extra_state
.get('webvtt_local', 0))
376 elif isinstance(block
, webvtt
.HeaderBlock
):
378 # XXX: this should probably be silent as well
379 # or verify that all segments contain the same data
380 self
.report_warning(bug_reports_message(
381 'Discarding a %s block found in the middle of the stream; '
382 'if the subtitles display incorrectly,'
383 % (type(block
).__name
__)))
385 block
.write_into(output
)
387 return output
.getvalue().encode('utf-8')
389 def append_fragment(frag_content
, frag_index
):
390 fatal
= frag_index
== 1 or not skip_unavailable_fragments
392 fragment_filename
= '%s-Frag%d' % (ctx
['tmpfilename'], frag_index
)
394 file, frag_sanitized
= sanitize_open(fragment_filename
, 'rb')
395 ctx
['fragment_filename_sanitized'] = frag_sanitized
397 frag_content
= pack_fragment(frag_content
, frag_index
)
398 self
._append
_fragment
(ctx
, frag_content
)
400 except EnvironmentError as ose
:
401 if ose
.errno
!= errno
.ENOENT
:
405 self
.report_skip_fragment(frag_index
)
408 ctx
['dest_stream'].close()
410 'fragment %s not found, unable to continue' % frag_index
)
414 self
.report_skip_fragment(frag_index
)
417 ctx
['dest_stream'].close()
419 'fragment %s not found, unable to continue' % frag_index
)
422 max_workers
= self
.params
.get('concurrent_fragment_downloads', 1)
423 if can_threaded_download
and max_workers
> 1:
424 self
.report_warning('The download speed shown is only of one thread. This is a known issue')
425 with concurrent
.futures
.ThreadPoolExecutor(max_workers
) as pool
:
426 futures
= [pool
.submit(download_fragment
, fragment
) for fragment
in fragments
]
427 # timeout must be 0 to return instantly
428 done
, not_done
= concurrent
.futures
.wait(futures
, timeout
=0)
431 # Check every 1 second for KeyboardInterrupt
432 freshly_done
, not_done
= concurrent
.futures
.wait(not_done
, timeout
=1)
434 except KeyboardInterrupt:
435 for future
in not_done
:
437 # timeout must be none to cancel
438 concurrent
.futures
.wait(not_done
, timeout
=None)
439 raise KeyboardInterrupt
440 results
= [future
.result() for future
in futures
]
442 for frag_content
, frag_index
in results
:
443 result
= append_fragment(frag_content
, frag_index
)
447 for fragment
in fragments
:
448 frag_content
, frag_index
= download_fragment(fragment
)
449 result
= append_fragment(frag_content
, frag_index
)
453 self
._finish
_frag
_download
(ctx
)