6 from . import get_suitable_downloader
7 from .external
import FFmpegFD
8 from .fragment
import FragmentFD
10 from ..dependencies
import Cryptodome
13 parse_m3u8_attributes
,
21 class HlsFD(FragmentFD
):
23 Download segments in a m3u8 manifest. External downloaders can take over
24 the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
25 re-defining 'supports_manifest' function
31 def can_download(manifest
, info_dict
, allow_unplayable_formats
=False):
32 UNSUPPORTED_FEATURES
= [
33 # r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2]
35 # Live streams heuristic does not always work (e.g. geo restricted to Germany
36 # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
37 # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', # live streams [3]
39 # This heuristic also is not correct since segments may not be appended as well.
40 # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
41 # no segments will definitely be appended to the end of the playlist.
42 # r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of
43 # # event media playlists [4]
44 # r'#EXT-X-MAP:', # media initialization [5]
45 # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
46 # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
47 # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
48 # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
49 # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
51 if not allow_unplayable_formats
:
52 UNSUPPORTED_FEATURES
+= [
53 r
'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1]
57 yield not info_dict
.get('is_live')
58 for feature
in UNSUPPORTED_FEATURES
:
59 yield not re
.search(feature
, manifest
)
60 return all(check_results())
62 def real_download(self
, filename
, info_dict
):
63 man_url
= info_dict
['url']
64 self
.to_screen('[%s] Downloading m3u8 manifest' % self
.FD_NAME
)
66 urlh
= self
.ydl
.urlopen(self
._prepare
_url
(info_dict
, man_url
))
67 man_url
= urlh
.geturl()
68 s
= urlh
.read().decode('utf-8', 'ignore')
70 can_download
, message
= self
.can_download(s
, info_dict
, self
.params
.get('allow_unplayable_formats')), None
72 has_ffmpeg
= FFmpegFD
.available()
73 no_crypto
= not Cryptodome
.AES
and '#EXT-X-KEY:METHOD=AES-128' in s
74 if no_crypto
and has_ffmpeg
:
75 can_download
, message
= False, 'The stream has AES-128 encryption and pycryptodomex is not available'
77 message
= ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; '
78 'Decryption will be performed natively, but will be extremely slow')
79 elif info_dict
.get('extractor_key') == 'Generic' and re
.search(r
'(?m)#EXT-X-MEDIA-SEQUENCE:(?!0$)', s
):
80 install_ffmpeg
= '' if has_ffmpeg
else 'install ffmpeg and '
81 message
= ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
82 f
'please {install_ffmpeg}add "--downloader ffmpeg --hls-use-mpegts" to your command')
84 has_drm
= re
.search('|'.join([
85 r
'#EXT-X-FAXS-CM:', # Adobe Flash Access
86 r
'#EXT-X-(?:SESSION-)?KEY:.*?URI="skd://', # Apple FairPlay
88 if has_drm
and not self
.params
.get('allow_unplayable_formats'):
90 'This video is DRM protected; Try selecting another format with --format or '
91 'add --check-formats to automatically fallback to the next best format')
93 message
= message
or 'Unsupported features have been detected'
94 fd
= FFmpegFD(self
.ydl
, self
.params
)
95 self
.report_warning(f
'{message}; extraction will be delegated to {fd.get_basename()}')
96 return fd
.real_download(filename
, info_dict
)
98 self
.report_warning(message
)
100 is_webvtt
= info_dict
['ext'] == 'vtt'
102 real_downloader
= None # Packing the fragments is not currently supported for external downloader
104 real_downloader
= get_suitable_downloader(
105 info_dict
, self
.params
, None, protocol
='m3u8_frag_urls', to_stdout
=(filename
== '-'))
106 if real_downloader
and not real_downloader
.supports_manifest(s
):
107 real_downloader
= None
109 self
.to_screen(f
'[{self.FD_NAME}] Fragment downloads will be delegated to {real_downloader.get_basename()}')
111 def is_ad_fragment_start(s
):
112 return (s
.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
113 or s
.startswith('#UPLYNK-SEGMENT') and s
.endswith(',ad'))
115 def is_ad_fragment_end(s
):
116 return (s
.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s
117 or s
.startswith('#UPLYNK-SEGMENT') and s
.endswith(',segment'))
124 for line
in s
.splitlines():
128 if line
.startswith('#'):
129 if is_ad_fragment_start(line
):
131 elif is_ad_fragment_end(line
):
140 'filename': filename
,
141 'total_frags': media_frags
,
142 'ad_frags': ad_frags
,
146 self
._prepare
_external
_frag
_download
(ctx
)
148 self
._prepare
_and
_start
_frag
_download
(ctx
, info_dict
)
150 extra_state
= ctx
.setdefault('extra_state', {})
152 format_index
= info_dict
.get('format_index')
154 extra_param_to_segment_url
= info_dict
.get('extra_param_to_segment_url')
155 if extra_param_to_segment_url
:
156 extra_query
= urllib
.parse
.parse_qs(extra_param_to_segment_url
)
159 decrypt_info
= {'METHOD': 'NONE'}
160 external_aes_key
= traverse_obj(info_dict
, ('hls_aes', 'key'))
162 external_aes_key
= binascii
.unhexlify(remove_start(external_aes_key
, '0x'))
163 assert len(external_aes_key
) in (16, 24, 32), 'Invalid length for HLS AES-128 key'
164 external_aes_iv
= traverse_obj(info_dict
, ('hls_aes', 'iv'))
166 external_aes_iv
= binascii
.unhexlify(remove_start(external_aes_iv
, '0x').zfill(32))
168 discontinuity_count
= 0
171 for line
in s
.splitlines():
174 if not line
.startswith('#'):
175 if format_index
and discontinuity_count
!= format_index
:
180 if frag_index
<= ctx
['fragment_index']:
182 frag_url
= urljoin(man_url
, line
)
184 frag_url
= update_url_query(frag_url
, extra_query
)
187 'frag_index': frag_index
,
189 'decrypt_info': decrypt_info
,
190 'byte_range': byte_range
,
191 'media_sequence': media_sequence
,
195 elif line
.startswith('#EXT-X-MAP'):
196 if format_index
and discontinuity_count
!= format_index
:
200 'Initialization fragment found after media fragments, unable to download')
203 map_info
= parse_m3u8_attributes(line
[11:])
204 frag_url
= urljoin(man_url
, map_info
.get('URI'))
206 frag_url
= update_url_query(frag_url
, extra_query
)
208 if map_info
.get('BYTERANGE'):
209 splitted_byte_range
= map_info
.get('BYTERANGE').split('@')
210 sub_range_start
= int(splitted_byte_range
[1]) if len(splitted_byte_range
) == 2 else byte_range
['end']
212 'start': sub_range_start
,
213 'end': sub_range_start
+ int(splitted_byte_range
[0]),
217 'frag_index': frag_index
,
219 'decrypt_info': decrypt_info
,
220 'byte_range': byte_range
,
221 'media_sequence': media_sequence
225 elif line
.startswith('#EXT-X-KEY'):
226 decrypt_url
= decrypt_info
.get('URI')
227 decrypt_info
= parse_m3u8_attributes(line
[11:])
228 if decrypt_info
['METHOD'] == 'AES-128':
230 decrypt_info
['IV'] = external_aes_iv
231 elif 'IV' in decrypt_info
:
232 decrypt_info
['IV'] = binascii
.unhexlify(decrypt_info
['IV'][2:].zfill(32))
234 decrypt_info
['KEY'] = external_aes_key
236 decrypt_info
['URI'] = urljoin(man_url
, decrypt_info
['URI'])
238 decrypt_info
['URI'] = update_url_query(decrypt_info
['URI'], extra_query
)
239 if decrypt_url
!= decrypt_info
['URI']:
240 decrypt_info
['KEY'] = None
242 elif line
.startswith('#EXT-X-MEDIA-SEQUENCE'):
243 media_sequence
= int(line
[22:])
244 elif line
.startswith('#EXT-X-BYTERANGE'):
245 splitted_byte_range
= line
[17:].split('@')
246 sub_range_start
= int(splitted_byte_range
[1]) if len(splitted_byte_range
) == 2 else byte_range
['end']
248 'start': sub_range_start
,
249 'end': sub_range_start
+ int(splitted_byte_range
[0]),
251 elif is_ad_fragment_start(line
):
253 elif is_ad_fragment_end(line
):
255 elif line
.startswith('#EXT-X-DISCONTINUITY'):
256 discontinuity_count
+= 1
259 # We only download the first fragment during the test
260 if self
.params
.get('test', False):
261 fragments
= [fragments
[0] if fragments
else None]
264 info_dict
['fragments'] = fragments
265 fd
= real_downloader(self
.ydl
, self
.params
)
266 # TODO: Make progress updates work without hooking twice
267 # for ph in self._progress_hooks:
268 # fd.add_progress_hook(ph)
269 return fd
.real_download(filename
, info_dict
)
272 def pack_fragment(frag_content
, frag_index
):
273 output
= io
.StringIO()
277 for block
in webvtt
.parse_fragment(frag_content
):
278 if isinstance(block
, webvtt
.CueBlock
):
279 extra_state
['webvtt_mpegts_last'] = mpegts_last
281 extra_state
['webvtt_mpegts_adjust'] += 1
283 block
.start
+= adjust
286 dedup_window
= extra_state
.setdefault('webvtt_dedup_window', [])
292 while i
< len(dedup_window
):
293 wcue
= dedup_window
[i
]
294 wblock
= webvtt
.CueBlock
.from_json(wcue
)
296 if wblock
.hinges(block
):
297 wcue
['end'] = block
.end
303 if wblock
.end
> block
.start
:
310 dedup_window
.append(block
.as_json
)
312 block
.write_into(output
)
314 # we only emit cues once they fall out of the duplicate window
316 elif isinstance(block
, webvtt
.Magic
):
317 # take care of MPEG PES timestamp overflow
318 if block
.mpegts
is None:
320 extra_state
.setdefault('webvtt_mpegts_adjust', 0)
321 block
.mpegts
+= extra_state
['webvtt_mpegts_adjust'] << 33
322 if block
.mpegts
< extra_state
.get('webvtt_mpegts_last', 0):
324 block
.mpegts
+= 1 << 33
325 mpegts_last
= block
.mpegts
328 extra_state
['webvtt_mpegts'] = block
.mpegts
or 0
329 extra_state
['webvtt_local'] = block
.local
or 0
330 # XXX: block.local = block.mpegts = None ?
332 if block
.mpegts
is not None and block
.local
is not None:
334 (block
.mpegts
- extra_state
.get('webvtt_mpegts', 0))
335 - (block
.local
- extra_state
.get('webvtt_local', 0))
338 elif isinstance(block
, webvtt
.HeaderBlock
):
340 # XXX: this should probably be silent as well
341 # or verify that all segments contain the same data
342 self
.report_warning(bug_reports_message(
343 'Discarding a %s block found in the middle of the stream; '
344 'if the subtitles display incorrectly,'
345 % (type(block
).__name
__)))
347 block
.write_into(output
)
349 return output
.getvalue().encode()
352 dedup_window
= extra_state
.get('webvtt_dedup_window')
356 output
= io
.StringIO()
357 for cue
in dedup_window
:
358 webvtt
.CueBlock
.from_json(cue
).write_into(output
)
360 return output
.getvalue().encode()
362 self
.download_and_append_fragments(
363 ctx
, fragments
, info_dict
, pack_func
=pack_fragment
, finish_func
=fin_fragments
)
365 return self
.download_and_append_fragments(ctx
, fragments
, info_dict
)