]>
Commit | Line | Data |
---|---|---|
1 | import io | |
2 | import itertools | |
3 | import struct | |
4 | import time | |
5 | import urllib.error | |
6 | ||
7 | from .fragment import FragmentFD | |
8 | from ..compat import ( | |
9 | compat_b64decode, | |
10 | compat_etree_fromstring, | |
11 | compat_urllib_parse_urlparse, | |
12 | compat_urlparse, | |
13 | ) | |
14 | from ..utils import fix_xml_ampersands, xpath_text | |
15 | ||
16 | ||
17 | class DataTruncatedError(Exception): | |
18 | pass | |
19 | ||
20 | ||
21 | class FlvReader(io.BytesIO): | |
22 | """ | |
23 | Reader for Flv files | |
24 | The file format is documented in https://www.adobe.com/devnet/f4v.html | |
25 | """ | |
26 | ||
27 | def read_bytes(self, n): | |
28 | data = self.read(n) | |
29 | if len(data) < n: | |
30 | raise DataTruncatedError( | |
31 | 'FlvReader error: need %d bytes while only %d bytes got' % ( | |
32 | n, len(data))) | |
33 | return data | |
34 | ||
35 | # Utility functions for reading numbers and strings | |
36 | def read_unsigned_long_long(self): | |
37 | return struct.unpack('!Q', self.read_bytes(8))[0] | |
38 | ||
39 | def read_unsigned_int(self): | |
40 | return struct.unpack('!I', self.read_bytes(4))[0] | |
41 | ||
42 | def read_unsigned_char(self): | |
43 | return struct.unpack('!B', self.read_bytes(1))[0] | |
44 | ||
45 | def read_string(self): | |
46 | res = b'' | |
47 | while True: | |
48 | char = self.read_bytes(1) | |
49 | if char == b'\x00': | |
50 | break | |
51 | res += char | |
52 | return res | |
53 | ||
54 | def read_box_info(self): | |
55 | """ | |
56 | Read a box and return the info as a tuple: (box_size, box_type, box_data) | |
57 | """ | |
58 | real_size = size = self.read_unsigned_int() | |
59 | box_type = self.read_bytes(4) | |
60 | header_end = 8 | |
61 | if size == 1: | |
62 | real_size = self.read_unsigned_long_long() | |
63 | header_end = 16 | |
64 | return real_size, box_type, self.read_bytes(real_size - header_end) | |
65 | ||
66 | def read_asrt(self): | |
67 | # version | |
68 | self.read_unsigned_char() | |
69 | # flags | |
70 | self.read_bytes(3) | |
71 | quality_entry_count = self.read_unsigned_char() | |
72 | # QualityEntryCount | |
73 | for i in range(quality_entry_count): | |
74 | self.read_string() | |
75 | ||
76 | segment_run_count = self.read_unsigned_int() | |
77 | segments = [] | |
78 | for i in range(segment_run_count): | |
79 | first_segment = self.read_unsigned_int() | |
80 | fragments_per_segment = self.read_unsigned_int() | |
81 | segments.append((first_segment, fragments_per_segment)) | |
82 | ||
83 | return { | |
84 | 'segment_run': segments, | |
85 | } | |
86 | ||
87 | def read_afrt(self): | |
88 | # version | |
89 | self.read_unsigned_char() | |
90 | # flags | |
91 | self.read_bytes(3) | |
92 | # time scale | |
93 | self.read_unsigned_int() | |
94 | ||
95 | quality_entry_count = self.read_unsigned_char() | |
96 | # QualitySegmentUrlModifiers | |
97 | for i in range(quality_entry_count): | |
98 | self.read_string() | |
99 | ||
100 | fragments_count = self.read_unsigned_int() | |
101 | fragments = [] | |
102 | for i in range(fragments_count): | |
103 | first = self.read_unsigned_int() | |
104 | first_ts = self.read_unsigned_long_long() | |
105 | duration = self.read_unsigned_int() | |
106 | if duration == 0: | |
107 | discontinuity_indicator = self.read_unsigned_char() | |
108 | else: | |
109 | discontinuity_indicator = None | |
110 | fragments.append({ | |
111 | 'first': first, | |
112 | 'ts': first_ts, | |
113 | 'duration': duration, | |
114 | 'discontinuity_indicator': discontinuity_indicator, | |
115 | }) | |
116 | ||
117 | return { | |
118 | 'fragments': fragments, | |
119 | } | |
120 | ||
121 | def read_abst(self): | |
122 | # version | |
123 | self.read_unsigned_char() | |
124 | # flags | |
125 | self.read_bytes(3) | |
126 | ||
127 | self.read_unsigned_int() # BootstrapinfoVersion | |
128 | # Profile,Live,Update,Reserved | |
129 | flags = self.read_unsigned_char() | |
130 | live = flags & 0x20 != 0 | |
131 | # time scale | |
132 | self.read_unsigned_int() | |
133 | # CurrentMediaTime | |
134 | self.read_unsigned_long_long() | |
135 | # SmpteTimeCodeOffset | |
136 | self.read_unsigned_long_long() | |
137 | ||
138 | self.read_string() # MovieIdentifier | |
139 | server_count = self.read_unsigned_char() | |
140 | # ServerEntryTable | |
141 | for i in range(server_count): | |
142 | self.read_string() | |
143 | quality_count = self.read_unsigned_char() | |
144 | # QualityEntryTable | |
145 | for i in range(quality_count): | |
146 | self.read_string() | |
147 | # DrmData | |
148 | self.read_string() | |
149 | # MetaData | |
150 | self.read_string() | |
151 | ||
152 | segments_count = self.read_unsigned_char() | |
153 | segments = [] | |
154 | for i in range(segments_count): | |
155 | box_size, box_type, box_data = self.read_box_info() | |
156 | assert box_type == b'asrt' | |
157 | segment = FlvReader(box_data).read_asrt() | |
158 | segments.append(segment) | |
159 | fragments_run_count = self.read_unsigned_char() | |
160 | fragments = [] | |
161 | for i in range(fragments_run_count): | |
162 | box_size, box_type, box_data = self.read_box_info() | |
163 | assert box_type == b'afrt' | |
164 | fragments.append(FlvReader(box_data).read_afrt()) | |
165 | ||
166 | return { | |
167 | 'segments': segments, | |
168 | 'fragments': fragments, | |
169 | 'live': live, | |
170 | } | |
171 | ||
172 | def read_bootstrap_info(self): | |
173 | total_size, box_type, box_data = self.read_box_info() | |
174 | assert box_type == b'abst' | |
175 | return FlvReader(box_data).read_abst() | |
176 | ||
177 | ||
178 | def read_bootstrap_info(bootstrap_bytes): | |
179 | return FlvReader(bootstrap_bytes).read_bootstrap_info() | |
180 | ||
181 | ||
182 | def build_fragments_list(boot_info): | |
183 | """ Return a list of (segment, fragment) for each fragment in the video """ | |
184 | res = [] | |
185 | segment_run_table = boot_info['segments'][0] | |
186 | fragment_run_entry_table = boot_info['fragments'][0]['fragments'] | |
187 | first_frag_number = fragment_run_entry_table[0]['first'] | |
188 | fragments_counter = itertools.count(first_frag_number) | |
189 | for segment, fragments_count in segment_run_table['segment_run']: | |
190 | # In some live HDS streams (for example Rai), `fragments_count` is | |
191 | # abnormal and causing out-of-memory errors. It's OK to change the | |
192 | # number of fragments for live streams as they are updated periodically | |
193 | if fragments_count == 4294967295 and boot_info['live']: | |
194 | fragments_count = 2 | |
195 | for _ in range(fragments_count): | |
196 | res.append((segment, next(fragments_counter))) | |
197 | ||
198 | if boot_info['live']: | |
199 | res = res[-2:] | |
200 | ||
201 | return res | |
202 | ||
203 | ||
204 | def write_unsigned_int(stream, val): | |
205 | stream.write(struct.pack('!I', val)) | |
206 | ||
207 | ||
208 | def write_unsigned_int_24(stream, val): | |
209 | stream.write(struct.pack('!I', val)[1:]) | |
210 | ||
211 | ||
212 | def write_flv_header(stream): | |
213 | """Writes the FLV header to stream""" | |
214 | # FLV header | |
215 | stream.write(b'FLV\x01') | |
216 | stream.write(b'\x05') | |
217 | stream.write(b'\x00\x00\x00\x09') | |
218 | stream.write(b'\x00\x00\x00\x00') | |
219 | ||
220 | ||
221 | def write_metadata_tag(stream, metadata): | |
222 | """Writes optional metadata tag to stream""" | |
223 | SCRIPT_TAG = b'\x12' | |
224 | FLV_TAG_HEADER_LEN = 11 | |
225 | ||
226 | if metadata: | |
227 | stream.write(SCRIPT_TAG) | |
228 | write_unsigned_int_24(stream, len(metadata)) | |
229 | stream.write(b'\x00\x00\x00\x00\x00\x00\x00') | |
230 | stream.write(metadata) | |
231 | write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata)) | |
232 | ||
233 | ||
234 | def remove_encrypted_media(media): | |
235 | return list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib | |
236 | and 'drmAdditionalHeaderSetId' not in e.attrib, | |
237 | media)) | |
238 | ||
239 | ||
240 | def _add_ns(prop, ver=1): | |
241 | return '{http://ns.adobe.com/f4m/%d.0}%s' % (ver, prop) | |
242 | ||
243 | ||
244 | def get_base_url(manifest): | |
245 | base_url = xpath_text( | |
246 | manifest, [_add_ns('baseURL'), _add_ns('baseURL', 2)], | |
247 | 'base URL', default=None) | |
248 | if base_url: | |
249 | base_url = base_url.strip() | |
250 | return base_url | |
251 | ||
252 | ||
253 | class F4mFD(FragmentFD): | |
254 | """ | |
255 | A downloader for f4m manifests or AdobeHDS. | |
256 | """ | |
257 | ||
258 | def _get_unencrypted_media(self, doc): | |
259 | media = doc.findall(_add_ns('media')) | |
260 | if not media: | |
261 | self.report_error('No media found') | |
262 | if not self.params.get('allow_unplayable_formats'): | |
263 | for e in (doc.findall(_add_ns('drmAdditionalHeader')) | |
264 | + doc.findall(_add_ns('drmAdditionalHeaderSet'))): | |
265 | # If id attribute is missing it's valid for all media nodes | |
266 | # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute | |
267 | if 'id' not in e.attrib: | |
268 | self.report_error('Missing ID in f4m DRM') | |
269 | media = remove_encrypted_media(media) | |
270 | if not media: | |
271 | self.report_error('Unsupported DRM') | |
272 | return media | |
273 | ||
274 | def _get_bootstrap_from_url(self, bootstrap_url): | |
275 | bootstrap = self.ydl.urlopen(bootstrap_url).read() | |
276 | return read_bootstrap_info(bootstrap) | |
277 | ||
278 | def _update_live_fragments(self, bootstrap_url, latest_fragment): | |
279 | fragments_list = [] | |
280 | retries = 30 | |
281 | while (not fragments_list) and (retries > 0): | |
282 | boot_info = self._get_bootstrap_from_url(bootstrap_url) | |
283 | fragments_list = build_fragments_list(boot_info) | |
284 | fragments_list = [f for f in fragments_list if f[1] > latest_fragment] | |
285 | if not fragments_list: | |
286 | # Retry after a while | |
287 | time.sleep(5.0) | |
288 | retries -= 1 | |
289 | ||
290 | if not fragments_list: | |
291 | self.report_error('Failed to update fragments') | |
292 | ||
293 | return fragments_list | |
294 | ||
295 | def _parse_bootstrap_node(self, node, base_url): | |
296 | # Sometimes non empty inline bootstrap info can be specified along | |
297 | # with bootstrap url attribute (e.g. dummy inline bootstrap info | |
298 | # contains whitespace characters in [1]). We will prefer bootstrap | |
299 | # url over inline bootstrap info when present. | |
300 | # 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m | |
301 | bootstrap_url = node.get('url') | |
302 | if bootstrap_url: | |
303 | bootstrap_url = compat_urlparse.urljoin( | |
304 | base_url, bootstrap_url) | |
305 | boot_info = self._get_bootstrap_from_url(bootstrap_url) | |
306 | else: | |
307 | bootstrap_url = None | |
308 | bootstrap = compat_b64decode(node.text) | |
309 | boot_info = read_bootstrap_info(bootstrap) | |
310 | return boot_info, bootstrap_url | |
311 | ||
312 | def real_download(self, filename, info_dict): | |
313 | man_url = info_dict['url'] | |
314 | requested_bitrate = info_dict.get('tbr') | |
315 | self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME) | |
316 | ||
317 | urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) | |
318 | man_url = urlh.geturl() | |
319 | # Some manifests may be malformed, e.g. prosiebensat1 generated manifests | |
320 | # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244 | |
321 | # and https://github.com/ytdl-org/youtube-dl/issues/7823) | |
322 | manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip() | |
323 | ||
324 | doc = compat_etree_fromstring(manifest) | |
325 | formats = [(int(f.attrib.get('bitrate', -1)), f) | |
326 | for f in self._get_unencrypted_media(doc)] | |
327 | if requested_bitrate is None or len(formats) == 1: | |
328 | # get the best format | |
329 | formats = sorted(formats, key=lambda f: f[0]) | |
330 | rate, media = formats[-1] | |
331 | else: | |
332 | rate, media = list(filter( | |
333 | lambda f: int(f[0]) == requested_bitrate, formats))[0] | |
334 | ||
335 | # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec. | |
336 | man_base_url = get_base_url(doc) or man_url | |
337 | ||
338 | base_url = compat_urlparse.urljoin(man_base_url, media.attrib['url']) | |
339 | bootstrap_node = doc.find(_add_ns('bootstrapInfo')) | |
340 | boot_info, bootstrap_url = self._parse_bootstrap_node( | |
341 | bootstrap_node, man_base_url) | |
342 | live = boot_info['live'] | |
343 | metadata_node = media.find(_add_ns('metadata')) | |
344 | if metadata_node is not None: | |
345 | metadata = compat_b64decode(metadata_node.text) | |
346 | else: | |
347 | metadata = None | |
348 | ||
349 | fragments_list = build_fragments_list(boot_info) | |
350 | test = self.params.get('test', False) | |
351 | if test: | |
352 | # We only download the first fragment | |
353 | fragments_list = fragments_list[:1] | |
354 | total_frags = len(fragments_list) | |
355 | # For some akamai manifests we'll need to add a query to the fragment url | |
356 | akamai_pv = xpath_text(doc, _add_ns('pv-2.0')) | |
357 | ||
358 | ctx = { | |
359 | 'filename': filename, | |
360 | 'total_frags': total_frags, | |
361 | 'live': bool(live), | |
362 | } | |
363 | ||
364 | self._prepare_frag_download(ctx) | |
365 | ||
366 | dest_stream = ctx['dest_stream'] | |
367 | ||
368 | if ctx['complete_frags_downloaded_bytes'] == 0: | |
369 | write_flv_header(dest_stream) | |
370 | if not live: | |
371 | write_metadata_tag(dest_stream, metadata) | |
372 | ||
373 | base_url_parsed = compat_urllib_parse_urlparse(base_url) | |
374 | ||
375 | self._start_frag_download(ctx, info_dict) | |
376 | ||
377 | frag_index = 0 | |
378 | while fragments_list: | |
379 | seg_i, frag_i = fragments_list.pop(0) | |
380 | frag_index += 1 | |
381 | if frag_index <= ctx['fragment_index']: | |
382 | continue | |
383 | name = 'Seg%d-Frag%d' % (seg_i, frag_i) | |
384 | query = [] | |
385 | if base_url_parsed.query: | |
386 | query.append(base_url_parsed.query) | |
387 | if akamai_pv: | |
388 | query.append(akamai_pv.strip(';')) | |
389 | if info_dict.get('extra_param_to_segment_url'): | |
390 | query.append(info_dict['extra_param_to_segment_url']) | |
391 | url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query)) | |
392 | try: | |
393 | success = self._download_fragment(ctx, url_parsed.geturl(), info_dict) | |
394 | if not success: | |
395 | return False | |
396 | down_data = self._read_fragment(ctx) | |
397 | reader = FlvReader(down_data) | |
398 | while True: | |
399 | try: | |
400 | _, box_type, box_data = reader.read_box_info() | |
401 | except DataTruncatedError: | |
402 | if test: | |
403 | # In tests, segments may be truncated, and thus | |
404 | # FlvReader may not be able to parse the whole | |
405 | # chunk. If so, write the segment as is | |
406 | # See https://github.com/ytdl-org/youtube-dl/issues/9214 | |
407 | dest_stream.write(down_data) | |
408 | break | |
409 | raise | |
410 | if box_type == b'mdat': | |
411 | self._append_fragment(ctx, box_data) | |
412 | break | |
413 | except urllib.error.HTTPError as err: | |
414 | if live and (err.code == 404 or err.code == 410): | |
415 | # We didn't keep up with the live window. Continue | |
416 | # with the next available fragment. | |
417 | msg = 'Fragment %d unavailable' % frag_i | |
418 | self.report_warning(msg) | |
419 | fragments_list = [] | |
420 | else: | |
421 | raise | |
422 | ||
423 | if not fragments_list and not test and live and bootstrap_url: | |
424 | fragments_list = self._update_live_fragments(bootstrap_url, frag_i) | |
425 | total_frags += len(fragments_list) | |
426 | if fragments_list and (fragments_list[0][1] > frag_i + 1): | |
427 | msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1)) | |
428 | self.report_warning(msg) | |
429 | ||
430 | self._finish_frag_download(ctx, info_dict) | |
431 | ||
432 | return True |