]>
Commit | Line | Data |
---|---|---|
f0b5d6af PH |
1 | from __future__ import unicode_literals |
2 | ||
f0b5d6af | 3 | import re |
e154c651 | 4 | import binascii |
5 | try: | |
6 | from Crypto.Cipher import AES | |
7 | can_decrypt_frag = True | |
8 | except ImportError: | |
9 | can_decrypt_frag = False | |
3bc2ddcc | 10 | |
5219cb3e | 11 | from ..downloader import _get_real_downloader |
f9a5affa | 12 | from .fragment import FragmentFD |
0d66bd0e | 13 | from .external import FFmpegFD |
f9a5affa | 14 | |
e154c651 | 15 | from ..compat import ( |
25afc2a7 | 16 | compat_urllib_error, |
e154c651 | 17 | compat_urlparse, |
18 | compat_struct_pack, | |
19 | ) | |
1cc79574 | 20 | from ..utils import ( |
e154c651 | 21 | parse_m3u8_attributes, |
aaf44a2f | 22 | update_url_query, |
3bc2ddcc JMF |
23 | ) |
24 | ||
25 | ||
12b84ac8 | 26 | class HlsFD(FragmentFD): |
27 | """ A limited implementation that does not require ffmpeg """ | |
f0b5d6af | 28 | |
f9a5affa S |
29 | FD_NAME = 'hlsnative' |
30 | ||
0d66bd0e | 31 | @staticmethod |
63ad4d43 | 32 | def can_download(manifest, info_dict, allow_unplayable_formats=False): |
33 | UNSUPPORTED_FEATURES = [ | |
f5974637 | 34 | # r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2] |
1e236d7e | 35 | |
c15c47d1 S |
36 | # Live streams heuristic does not always work (e.g. geo restricted to Germany |
37 | # http://hls-geo.daserste.de/i/videoportal/Film/c_620000/622873/format,716451,716457,716450,716458,716459,.mp4.csmil/index_4_av.m3u8?null=0) | |
2937590e | 38 | # r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', # live streams [3] |
1e236d7e S |
39 | |
40 | # This heuristic also is not correct since segments may not be appended as well. | |
633b444f S |
41 | # Twitch vods of finished streams have EXT-X-PLAYLIST-TYPE:EVENT despite |
42 | # no segments will definitely be appended to the end of the playlist. | |
1e236d7e | 43 | # r'#EXT-X-PLAYLIST-TYPE:EVENT', # media segments may be appended to the end of |
51c4d85c | 44 | # # event media playlists [4] |
29f7c58a | 45 | r'#EXT-X-MAP:', # media initialization [5] |
1e236d7e | 46 | |
0d66bd0e S |
47 | # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.4 |
48 | # 2. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.2 | |
49 | # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2 | |
6104cc29 | 50 | # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5 |
29f7c58a | 51 | # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5 |
63ad4d43 | 52 | ] |
53 | if not allow_unplayable_formats: | |
54 | UNSUPPORTED_FEATURES += [ | |
55 | r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1] | |
56 | ] | |
e154c651 | 57 | check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES] |
f5974637 RA |
58 | is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest |
59 | check_results.append(can_decrypt_frag or not is_aes128_enc) | |
60 | check_results.append(not (is_aes128_enc and r'#EXT-X-BYTERANGE' in manifest)) | |
2bfaf89b | 61 | check_results.append(not info_dict.get('is_live')) |
e154c651 | 62 | return all(check_results) |
0d66bd0e | 63 | |
f0b5d6af | 64 | def real_download(self, filename, info_dict): |
f9a5affa S |
65 | man_url = info_dict['url'] |
66 | self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME) | |
69035555 | 67 | |
c5a49ff0 S |
68 | urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) |
69 | man_url = urlh.geturl() | |
70 | s = urlh.read().decode('utf-8', 'ignore') | |
0d66bd0e | 71 | |
63ad4d43 | 72 | if not self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')): |
c712b16d | 73 | if info_dict.get('extra_param_to_segment_url') or info_dict.get('_decryption_key_url'): |
bfa1073e RA |
74 | self.report_error('pycrypto not found. Please install it.') |
75 | return False | |
2bfaf89b RA |
76 | self.report_warning( |
77 | 'hlsnative has detected features it does not support, ' | |
78 | 'extraction will be delegated to ffmpeg') | |
79 | fd = FFmpegFD(self.ydl, self.params) | |
5219cb3e | 80 | # TODO: Make progress updates work without hooking twice |
81 | # for ph in self._progress_hooks: | |
82 | # fd.add_progress_hook(ph) | |
2bfaf89b | 83 | return fd.real_download(filename, info_dict) |
0d66bd0e | 84 | |
5219cb3e | 85 | real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None) |
86 | ||
f1ab3b7d | 87 | def is_ad_fragment_start(s): |
3089bc74 S |
88 | return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=ad' in s |
89 | or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',ad')) | |
74c42d9e | 90 | |
f1ab3b7d | 91 | def is_ad_fragment_end(s): |
3089bc74 S |
92 | return (s.startswith('#ANVATO-SEGMENT-INFO') and 'type=master' in s |
93 | or s.startswith('#UPLYNK-SEGMENT') and s.endswith(',segment')) | |
f1ab3b7d | 94 | |
5219cb3e | 95 | fragment_urls = [] |
96 | ||
74c42d9e S |
97 | media_frags = 0 |
98 | ad_frags = 0 | |
99 | ad_frag_next = False | |
f0b5d6af PH |
100 | for line in s.splitlines(): |
101 | line = line.strip() | |
74c42d9e S |
102 | if not line: |
103 | continue | |
104 | if line.startswith('#'): | |
f1ab3b7d | 105 | if is_ad_fragment_start(line): |
a9ee4f6e | 106 | ad_frag_next = True |
f1ab3b7d RA |
107 | elif is_ad_fragment_end(line): |
108 | ad_frag_next = False | |
74c42d9e S |
109 | continue |
110 | if ad_frag_next: | |
f1ab3b7d | 111 | ad_frags += 1 |
74c42d9e S |
112 | continue |
113 | media_frags += 1 | |
f0b5d6af | 114 | |
f9a5affa | 115 | ctx = { |
f0b5d6af | 116 | 'filename': filename, |
74c42d9e S |
117 | 'total_frags': media_frags, |
118 | 'ad_frags': ad_frags, | |
f9a5affa S |
119 | } |
120 | ||
5219cb3e | 121 | if real_downloader: |
122 | self._prepare_external_frag_download(ctx) | |
123 | else: | |
124 | self._prepare_and_start_frag_download(ctx) | |
f9a5affa | 125 | |
25afc2a7 S |
126 | fragment_retries = self.params.get('fragment_retries', 0) |
127 | skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True) | |
128 | test = self.params.get('test', False) | |
129 | ||
b8079a40 | 130 | extra_query = None |
aaf44a2f | 131 | extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url') |
b8079a40 RA |
132 | if extra_param_to_segment_url: |
133 | extra_query = compat_urlparse.parse_qs(extra_param_to_segment_url) | |
e154c651 | 134 | i = 0 |
135 | media_sequence = 0 | |
136 | decrypt_info = {'METHOD': 'NONE'} | |
7620cd46 | 137 | key_list = [] |
f5974637 | 138 | byte_range = {} |
75a24854 | 139 | frag_index = 0 |
74c42d9e | 140 | ad_frag_next = False |
e154c651 | 141 | for line in s.splitlines(): |
142 | line = line.strip() | |
143 | if line: | |
144 | if not line.startswith('#'): | |
74c42d9e | 145 | if ad_frag_next: |
74c42d9e | 146 | continue |
75a24854 | 147 | frag_index += 1 |
3e0304fe | 148 | if frag_index <= ctx['fragment_index']: |
75a24854 | 149 | continue |
e154c651 | 150 | frag_url = ( |
151 | line | |
152 | if re.match(r'^https?://', line) | |
153 | else compat_urlparse.urljoin(man_url, line)) | |
b8079a40 RA |
154 | if extra_query: |
155 | frag_url = update_url_query(frag_url, extra_query) | |
5219cb3e | 156 | |
157 | if real_downloader: | |
158 | fragment_urls.append(frag_url) | |
159 | continue | |
160 | ||
25afc2a7 | 161 | count = 0 |
f5974637 RA |
162 | headers = info_dict.get('http_headers', {}) |
163 | if byte_range: | |
6e65a2a6 | 164 | headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1) |
25afc2a7 S |
165 | while count <= fragment_retries: |
166 | try: | |
75a24854 RA |
167 | success, frag_content = self._download_fragment( |
168 | ctx, frag_url, info_dict, headers) | |
25afc2a7 S |
169 | if not success: |
170 | return False | |
25afc2a7 | 171 | break |
2e99cd30 | 172 | except compat_urllib_error.HTTPError as err: |
25afc2a7 S |
173 | # Unavailable (possibly temporary) fragments may be served. |
174 | # First we try to retry then either skip or abort. | |
067aa17e S |
175 | # See https://github.com/ytdl-org/youtube-dl/issues/10165, |
176 | # https://github.com/ytdl-org/youtube-dl/issues/10448). | |
25afc2a7 S |
177 | count += 1 |
178 | if count <= fragment_retries: | |
75a24854 | 179 | self.report_retry_fragment(err, frag_index, count, fragment_retries) |
25afc2a7 S |
180 | if count > fragment_retries: |
181 | if skip_unavailable_fragments: | |
182 | i += 1 | |
183 | media_sequence += 1 | |
75a24854 | 184 | self.report_skip_fragment(frag_index) |
25afc2a7 S |
185 | continue |
186 | self.report_error( | |
187 | 'giving up after %s fragment retries' % fragment_retries) | |
e154c651 | 188 | return False |
5219cb3e | 189 | |
e154c651 | 190 | if decrypt_info['METHOD'] == 'AES-128': |
8369a4fe | 191 | iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence) |
538d4f86 | 192 | decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen( |
c712b16d | 193 | self._prepare_url(info_dict, info_dict.get('_decryption_key_url') or decrypt_info['URI'])).read() |
00dd0cd5 | 194 | # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block |
195 | # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded, | |
196 | # not what it decrypts to. | |
197 | if not test: | |
198 | frag_content = AES.new( | |
199 | decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content) | |
75a24854 | 200 | self._append_fragment(ctx, frag_content) |
e154c651 | 201 | # We only download the first fragment during the test |
25afc2a7 | 202 | if test: |
e154c651 | 203 | break |
204 | i += 1 | |
205 | media_sequence += 1 | |
206 | elif line.startswith('#EXT-X-KEY'): | |
75a24854 | 207 | decrypt_url = decrypt_info.get('URI') |
e154c651 | 208 | decrypt_info = parse_m3u8_attributes(line[11:]) |
209 | if decrypt_info['METHOD'] == 'AES-128': | |
210 | if 'IV' in decrypt_info: | |
07ea9c9b | 211 | decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32)) |
e154c651 | 212 | if not re.match(r'^https?://', decrypt_info['URI']): |
8369a4fe S |
213 | decrypt_info['URI'] = compat_urlparse.urljoin( |
214 | man_url, decrypt_info['URI']) | |
b8079a40 RA |
215 | if extra_query: |
216 | decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query) | |
75a24854 RA |
217 | if decrypt_url != decrypt_info['URI']: |
218 | decrypt_info['KEY'] = None | |
7620cd46 | 219 | key_data = decrypt_info.copy() |
220 | key_data['INDEX'] = frag_index | |
221 | key_list.append(key_data) | |
222 | ||
e154c651 | 223 | elif line.startswith('#EXT-X-MEDIA-SEQUENCE'): |
224 | media_sequence = int(line[22:]) | |
f5974637 RA |
225 | elif line.startswith('#EXT-X-BYTERANGE'): |
226 | splitted_byte_range = line[17:].split('@') | |
227 | sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] | |
228 | byte_range = { | |
229 | 'start': sub_range_start, | |
230 | 'end': sub_range_start + int(splitted_byte_range[0]), | |
231 | } | |
f1ab3b7d | 232 | elif is_ad_fragment_start(line): |
74c42d9e | 233 | ad_frag_next = True |
f1ab3b7d RA |
234 | elif is_ad_fragment_end(line): |
235 | ad_frag_next = False | |
f9a5affa | 236 | |
5219cb3e | 237 | if real_downloader: |
238 | info_copy = info_dict.copy() | |
239 | info_copy['url_list'] = fragment_urls | |
7620cd46 | 240 | info_copy['key_list'] = key_list |
5219cb3e | 241 | fd = real_downloader(self.ydl, self.params) |
242 | # TODO: Make progress updates work without hooking twice | |
243 | # for ph in self._progress_hooks: | |
244 | # fd.add_progress_hook(ph) | |
245 | success = fd.real_download(filename, info_copy) | |
246 | if not success: | |
247 | return False | |
248 | else: | |
249 | self._finish_frag_download(ctx) | |
f0b5d6af | 250 | return True |