1 from __future__
import unicode_literals
6 from Crypto
.Cipher
import AES
7 can_decrypt_frag
= True
9 can_decrypt_frag
= False
11 from ..downloader
import _get_real_downloader
12 from .fragment
import FragmentFD
13 from .external
import FFmpegFD
15 from ..compat
import (
21 parse_m3u8_attributes
,
26 class HlsFD(FragmentFD
):
27 """ A limited implementation that does not require ffmpeg """
32 def can_download(manifest
, info_dict
, allow_unplayable_formats
=False, with_crypto
=can_decrypt_frag
):
33 UNSUPPORTED_FEATURES
= [
34 # r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2]
36 # Live streams heuristic does not always work (e.g. geo restricted to Germany
37 # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0)
38 # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', # live streams [3]
40 # This heuristic also is not correct since segments may not be appended as well.
41 # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite
42 # no segments will definitely be appended to the end of the playlist.
43 # r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of
44 # # event media playlists [4]
45 # r'#EXT-X-MAP:', # media initialization [5]
46 # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4
47 # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2
48 # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
49 # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
50 # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
52 if not allow_unplayable_formats
:
53 UNSUPPORTED_FEATURES
+= [
54 r
'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1]
56 check_results
= [not re
.search(feature
, manifest
) for feature
in UNSUPPORTED_FEATURES
]
57 is_aes128_enc
= '#EXT-X-KEY:METHOD=AES-128' in manifest
58 check_results
.append(with_crypto
or not is_aes128_enc
)
59 check_results
.append(not (is_aes128_enc
and r
'#EXT-X-BYTERANGE' in manifest
))
60 check_results
.append(not info_dict
.get('is_live'))
61 return all(check_results
)
63 def real_download(self
, filename
, info_dict
):
64 man_url
= info_dict
['url']
65 self
.to_screen('[%s] Downloading m3u8 manifest' % self
.FD_NAME
)
67 urlh
= self
.ydl
.urlopen(self
._prepare
_url
(info_dict
, man_url
))
68 man_url
= urlh
.geturl()
69 s
= urlh
.read().decode('utf-8', 'ignore')
71 if not self
.can_download(s
, info_dict
, self
.params
.get('allow_unplayable_formats')):
72 if info_dict
.get('extra_param_to_segment_url') or info_dict
.get('_decryption_key_url'):
73 self
.report_error('pycryptodome not found. Please install it.')
75 if self
.can_download(s
, info_dict
, with_crypto
=True):
76 self
.report_warning('pycryptodome is needed to download this file with hlsnative')
78 'hlsnative has detected features it does not support, '
79 'extraction will be delegated to ffmpeg')
80 fd
= FFmpegFD(self
.ydl
, self
.params
)
81 # TODO: Make progress updates work without hooking twice
82 # for ph in self._progress_hooks:
83 # fd.add_progress_hook(ph)
84 return fd
.real_download(filename
, info_dict
)
86 real_downloader
= _get_real_downloader(info_dict
, 'frag_urls', self
.params
, None)
88 def is_ad_fragment_start(s
):
89 return (s
.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s
90 or s
.startswith('#UPLYNK-SEGMENT') and s
.endswith(',ad'))
92 def is_ad_fragment_end(s
):
93 return (s
.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s
94 or s
.startswith('#UPLYNK-SEGMENT') and s
.endswith(',segment'))
101 for line
in s
.splitlines():
105 if line
.startswith('#'):
106 if is_ad_fragment_start(line
):
108 elif is_ad_fragment_end(line
):
117 'filename': filename
,
118 'total_frags': media_frags
,
119 'ad_frags': ad_frags
,
123 self
._prepare
_external
_frag
_download
(ctx
)
125 self
._prepare
_and
_start
_frag
_download
(ctx
)
127 fragment_retries
= self
.params
.get('fragment_retries', 0)
128 skip_unavailable_fragments
= self
.params
.get('skip_unavailable_fragments', True)
129 test
= self
.params
.get('test', False)
131 format_index
= info_dict
.get('format_index')
133 extra_param_to_segment_url
= info_dict
.get('extra_param_to_segment_url')
134 if extra_param_to_segment_url
:
135 extra_query
= compat_urlparse
.parse_qs(extra_param_to_segment_url
)
138 decrypt_info
= {'METHOD': 'NONE'}
141 discontinuity_count
= 0
144 for line
in s
.splitlines():
146 download_frag
= False
148 if not line
.startswith('#'):
149 if format_index
and discontinuity_count
!= format_index
:
154 if frag_index
<= ctx
['fragment_index']:
158 if re
.match(r
'^https?://', line
)
159 else compat_urlparse
.urljoin(man_url
, line
))
161 frag_url
= update_url_query(frag_url
, extra_query
)
164 fragment_urls
.append(frag_url
)
168 elif line
.startswith('#EXT-X-MAP'):
169 if format_index
and discontinuity_count
!= format_index
:
173 'initialization fragment found after media fragments, unable to download')
176 map_info
= parse_m3u8_attributes(line
[11:])
179 if re
.match(r
'^https?://', map_info
.get('URI'))
180 else compat_urlparse
.urljoin(man_url
, map_info
.get('URI')))
182 frag_url
= update_url_query(frag_url
, extra_query
)
184 fragment_urls
.append(frag_url
)
187 if map_info
.get('BYTERANGE'):
188 splitted_byte_range
= map_info
.get('BYTERANGE').split('@')
189 sub_range_start
= int(splitted_byte_range
[1]) if len(splitted_byte_range
) == 2 else byte_range
['end']
191 'start': sub_range_start
,
192 'end': sub_range_start
+ int(splitted_byte_range
[0]),
196 elif line
.startswith('#EXT-X-KEY'):
197 decrypt_url
= decrypt_info
.get('URI')
198 decrypt_info
= parse_m3u8_attributes(line
[11:])
199 if decrypt_info
['METHOD'] == 'AES-128':
200 if 'IV' in decrypt_info
:
201 decrypt_info
['IV'] = binascii
.unhexlify(decrypt_info
['IV'][2:].zfill(32))
202 if not re
.match(r
'^https?://', decrypt_info
['URI']):
203 decrypt_info
['URI'] = compat_urlparse
.urljoin(
204 man_url
, decrypt_info
['URI'])
206 decrypt_info
['URI'] = update_url_query(decrypt_info
['URI'], extra_query
)
207 if decrypt_url
!= decrypt_info
['URI']:
208 decrypt_info
['KEY'] = None
209 key_data
= decrypt_info
.copy()
210 key_data
['INDEX'] = frag_index
211 key_list
.append(key_data
)
213 elif line
.startswith('#EXT-X-MEDIA-SEQUENCE'):
214 media_sequence
= int(line
[22:])
215 elif line
.startswith('#EXT-X-BYTERANGE'):
216 splitted_byte_range
= line
[17:].split('@')
217 sub_range_start
= int(splitted_byte_range
[1]) if len(splitted_byte_range
) == 2 else byte_range
['end']
219 'start': sub_range_start
,
220 'end': sub_range_start
+ int(splitted_byte_range
[0]),
222 elif is_ad_fragment_start(line
):
224 elif is_ad_fragment_end(line
):
226 elif line
.startswith('#EXT-X-DISCONTINUITY'):
227 discontinuity_count
+= 1
231 headers
= info_dict
.get('http_headers', {})
233 headers
['Range'] = 'bytes=%d-%d' % (byte_range
['start'], byte_range
['end'] - 1)
234 while count
<= fragment_retries
:
236 success
, frag_content
= self
._download
_fragment
(
237 ctx
, frag_url
, info_dict
, headers
)
241 except compat_urllib_error
.HTTPError
as err
:
242 # Unavailable (possibly temporary) fragments may be served.
243 # First we try to retry then either skip or abort.
244 # See https://github.com/ytdl-org/youtube-dl/issues/10165,
245 # https://github.com/ytdl-org/youtube-dl/issues/10448).
247 if count
<= fragment_retries
:
248 self
.report_retry_fragment(err
, frag_index
, count
, fragment_retries
)
249 if count
> fragment_retries
:
250 if skip_unavailable_fragments
:
253 self
.report_skip_fragment(frag_index
)
256 'giving up after %s fragment retries' % fragment_retries
)
259 if decrypt_info
['METHOD'] == 'AES-128':
260 iv
= decrypt_info
.get('IV') or compat_struct_pack('>8xq', media_sequence
)
261 decrypt_info
['KEY'] = decrypt_info
.get('KEY') or self
.ydl
.urlopen(
262 self
._prepare
_url
(info_dict
, info_dict
.get('_decryption_key_url') or decrypt_info
['URI'])).read()
263 # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
264 # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
265 # not what it decrypts to.
267 frag_content
= AES
.new(
268 decrypt_info
['KEY'], AES
.MODE_CBC
, iv
).decrypt(frag_content
)
269 self
._append
_fragment
(ctx
, frag_content
)
270 # We only download the first fragment during the test
277 info_copy
= info_dict
.copy()
278 info_copy
['url_list'] = fragment_urls
279 info_copy
['key_list'] = key_list
280 fd
= real_downloader(self
.ydl
, self
.params
)
281 # TODO: Make progress updates work without hooking twice
282 # for ph in self._progress_hooks:
283 # fd.add_progress_hook(ph)
284 success
= fd
.real_download(filename
, info_copy
)
288 self
._finish
_frag
_download
(ctx
)