5 from .fragment
import FragmentFD
8 compat_etree_fromstring
,
12 compat_urllib_parse_urlparse
,
15 from ..utils
import fix_xml_ampersands
, xpath_text
18 class DataTruncatedError(Exception):
22 class FlvReader(io
.BytesIO
):
25 The file format is documented in https://www.adobe.com/devnet/f4v.html
28 def read_bytes(self
, n
):
31 raise DataTruncatedError(
32 'FlvReader error: need %d bytes while only %d bytes got' % (
36 # Utility functions for reading numbers and strings
37 def read_unsigned_long_long(self
):
38 return compat_struct_unpack('!Q', self
.read_bytes(8))[0]
40 def read_unsigned_int(self
):
41 return compat_struct_unpack('!I', self
.read_bytes(4))[0]
43 def read_unsigned_char(self
):
44 return compat_struct_unpack('!B', self
.read_bytes(1))[0]
46 def read_string(self
):
49 char
= self
.read_bytes(1)
55 def read_box_info(self
):
57 Read a box and return the info as a tuple: (box_size, box_type, box_data)
59 real_size
= size
= self
.read_unsigned_int()
60 box_type
= self
.read_bytes(4)
63 real_size
= self
.read_unsigned_long_long()
65 return real_size
, box_type
, self
.read_bytes(real_size
- header_end
)
69 self
.read_unsigned_char()
72 quality_entry_count
= self
.read_unsigned_char()
74 for i
in range(quality_entry_count
):
77 segment_run_count
= self
.read_unsigned_int()
79 for i
in range(segment_run_count
):
80 first_segment
= self
.read_unsigned_int()
81 fragments_per_segment
= self
.read_unsigned_int()
82 segments
.append((first_segment
, fragments_per_segment
))
85 'segment_run': segments
,
90 self
.read_unsigned_char()
94 self
.read_unsigned_int()
96 quality_entry_count
= self
.read_unsigned_char()
97 # QualitySegmentUrlModifiers
98 for i
in range(quality_entry_count
):
101 fragments_count
= self
.read_unsigned_int()
103 for i
in range(fragments_count
):
104 first
= self
.read_unsigned_int()
105 first_ts
= self
.read_unsigned_long_long()
106 duration
= self
.read_unsigned_int()
108 discontinuity_indicator
= self
.read_unsigned_char()
110 discontinuity_indicator
= None
114 'duration': duration
,
115 'discontinuity_indicator': discontinuity_indicator
,
119 'fragments': fragments
,
124 self
.read_unsigned_char()
128 self
.read_unsigned_int() # BootstrapinfoVersion
129 # Profile,Live,Update,Reserved
130 flags
= self
.read_unsigned_char()
131 live
= flags
& 0x20 != 0
133 self
.read_unsigned_int()
135 self
.read_unsigned_long_long()
136 # SmpteTimeCodeOffset
137 self
.read_unsigned_long_long()
139 self
.read_string() # MovieIdentifier
140 server_count
= self
.read_unsigned_char()
142 for i
in range(server_count
):
144 quality_count
= self
.read_unsigned_char()
146 for i
in range(quality_count
):
153 segments_count
= self
.read_unsigned_char()
155 for i
in range(segments_count
):
156 box_size
, box_type
, box_data
= self
.read_box_info()
157 assert box_type
== b
'asrt'
158 segment
= FlvReader(box_data
).read_asrt()
159 segments
.append(segment
)
160 fragments_run_count
= self
.read_unsigned_char()
162 for i
in range(fragments_run_count
):
163 box_size
, box_type
, box_data
= self
.read_box_info()
164 assert box_type
== b
'afrt'
165 fragments
.append(FlvReader(box_data
).read_afrt())
168 'segments': segments
,
169 'fragments': fragments
,
173 def read_bootstrap_info(self
):
174 total_size
, box_type
, box_data
= self
.read_box_info()
175 assert box_type
== b
'abst'
176 return FlvReader(box_data
).read_abst()
179 def read_bootstrap_info(bootstrap_bytes
):
180 return FlvReader(bootstrap_bytes
).read_bootstrap_info()
183 def build_fragments_list(boot_info
):
184 """ Return a list of (segment, fragment) for each fragment in the video """
186 segment_run_table
= boot_info
['segments'][0]
187 fragment_run_entry_table
= boot_info
['fragments'][0]['fragments']
188 first_frag_number
= fragment_run_entry_table
[0]['first']
189 fragments_counter
= itertools
.count(first_frag_number
)
190 for segment
, fragments_count
in segment_run_table
['segment_run']:
191 # In some live HDS streams (for example Rai), `fragments_count` is
192 # abnormal and causing out-of-memory errors. It's OK to change the
193 # number of fragments for live streams as they are updated periodically
194 if fragments_count
== 4294967295 and boot_info
['live']:
196 for _
in range(fragments_count
):
197 res
.append((segment
, next(fragments_counter
)))
199 if boot_info
['live']:
205 def write_unsigned_int(stream
, val
):
206 stream
.write(compat_struct_pack('!I', val
))
209 def write_unsigned_int_24(stream
, val
):
210 stream
.write(compat_struct_pack('!I', val
)[1:])
213 def write_flv_header(stream
):
214 """Writes the FLV header to stream"""
216 stream
.write(b
'FLV\x01')
217 stream
.write(b
'\x05')
218 stream
.write(b
'\x00\x00\x00\x09')
219 stream
.write(b
'\x00\x00\x00\x00')
222 def write_metadata_tag(stream
, metadata
):
223 """Writes optional metadata tag to stream"""
225 FLV_TAG_HEADER_LEN
= 11
228 stream
.write(SCRIPT_TAG
)
229 write_unsigned_int_24(stream
, len(metadata
))
230 stream
.write(b
'\x00\x00\x00\x00\x00\x00\x00')
231 stream
.write(metadata
)
232 write_unsigned_int(stream
, FLV_TAG_HEADER_LEN
+ len(metadata
))
235 def remove_encrypted_media(media
):
236 return list(filter(lambda e
: 'drmAdditionalHeaderId' not in e
.attrib
237 and 'drmAdditionalHeaderSetId' not in e
.attrib
,
241 def _add_ns(prop
, ver
=1):
242 return '{http://ns.adobe.com/f4m/%d.0}%s' % (ver
, prop
)
245 def get_base_url(manifest
):
246 base_url
= xpath_text(
247 manifest
, [_add_ns('baseURL'), _add_ns('baseURL', 2)],
248 'base URL', default
=None)
250 base_url
= base_url
.strip()
254 class F4mFD(FragmentFD
):
256 A downloader for f4m manifests or AdobeHDS.
261 def _get_unencrypted_media(self
, doc
):
262 media
= doc
.findall(_add_ns('media'))
264 self
.report_error('No media found')
265 if not self
.params
.get('allow_unplayable_formats'):
266 for e
in (doc
.findall(_add_ns('drmAdditionalHeader'))
267 + doc
.findall(_add_ns('drmAdditionalHeaderSet'))):
268 # If id attribute is missing it's valid for all media nodes
269 # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
270 if 'id' not in e
.attrib
:
271 self
.report_error('Missing ID in f4m DRM')
272 media
= remove_encrypted_media(media
)
274 self
.report_error('Unsupported DRM')
277 def _get_bootstrap_from_url(self
, bootstrap_url
):
278 bootstrap
= self
.ydl
.urlopen(bootstrap_url
).read()
279 return read_bootstrap_info(bootstrap
)
281 def _update_live_fragments(self
, bootstrap_url
, latest_fragment
):
284 while (not fragments_list
) and (retries
> 0):
285 boot_info
= self
._get
_bootstrap
_from
_url
(bootstrap_url
)
286 fragments_list
= build_fragments_list(boot_info
)
287 fragments_list
= [f
for f
in fragments_list
if f
[1] > latest_fragment
]
288 if not fragments_list
:
289 # Retry after a while
293 if not fragments_list
:
294 self
.report_error('Failed to update fragments')
296 return fragments_list
298 def _parse_bootstrap_node(self
, node
, base_url
):
299 # Sometimes non empty inline bootstrap info can be specified along
300 # with bootstrap url attribute (e.g. dummy inline bootstrap info
301 # contains whitespace characters in [1]). We will prefer bootstrap
302 # url over inline bootstrap info when present.
303 # 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m
304 bootstrap_url
= node
.get('url')
306 bootstrap_url
= compat_urlparse
.urljoin(
307 base_url
, bootstrap_url
)
308 boot_info
= self
._get
_bootstrap
_from
_url
(bootstrap_url
)
311 bootstrap
= compat_b64decode(node
.text
)
312 boot_info
= read_bootstrap_info(bootstrap
)
313 return boot_info
, bootstrap_url
315 def real_download(self
, filename
, info_dict
):
316 man_url
= info_dict
['url']
317 requested_bitrate
= info_dict
.get('tbr')
318 self
.to_screen('[%s] Downloading f4m manifest' % self
.FD_NAME
)
320 urlh
= self
.ydl
.urlopen(self
._prepare
_url
(info_dict
, man_url
))
321 man_url
= urlh
.geturl()
322 # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
323 # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244
324 # and https://github.com/ytdl-org/youtube-dl/issues/7823)
325 manifest
= fix_xml_ampersands(urlh
.read().decode('utf-8', 'ignore')).strip()
327 doc
= compat_etree_fromstring(manifest
)
328 formats
= [(int(f
.attrib
.get('bitrate', -1)), f
)
329 for f
in self
._get
_unencrypted
_media
(doc
)]
330 if requested_bitrate
is None or len(formats
) == 1:
331 # get the best format
332 formats
= sorted(formats
, key
=lambda f
: f
[0])
333 rate
, media
= formats
[-1]
335 rate
, media
= list(filter(
336 lambda f
: int(f
[0]) == requested_bitrate
, formats
))[0]
338 # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec.
339 man_base_url
= get_base_url(doc
) or man_url
341 base_url
= compat_urlparse
.urljoin(man_base_url
, media
.attrib
['url'])
342 bootstrap_node
= doc
.find(_add_ns('bootstrapInfo'))
343 boot_info
, bootstrap_url
= self
._parse
_bootstrap
_node
(
344 bootstrap_node
, man_base_url
)
345 live
= boot_info
['live']
346 metadata_node
= media
.find(_add_ns('metadata'))
347 if metadata_node
is not None:
348 metadata
= compat_b64decode(metadata_node
.text
)
352 fragments_list
= build_fragments_list(boot_info
)
353 test
= self
.params
.get('test', False)
355 # We only download the first fragment
356 fragments_list
= fragments_list
[:1]
357 total_frags
= len(fragments_list
)
358 # For some akamai manifests we'll need to add a query to the fragment url
359 akamai_pv
= xpath_text(doc
, _add_ns('pv-2.0'))
362 'filename': filename
,
363 'total_frags': total_frags
,
367 self
._prepare
_frag
_download
(ctx
)
369 dest_stream
= ctx
['dest_stream']
371 if ctx
['complete_frags_downloaded_bytes'] == 0:
372 write_flv_header(dest_stream
)
374 write_metadata_tag(dest_stream
, metadata
)
376 base_url_parsed
= compat_urllib_parse_urlparse(base_url
)
378 self
._start
_frag
_download
(ctx
, info_dict
)
381 while fragments_list
:
382 seg_i
, frag_i
= fragments_list
.pop(0)
384 if frag_index
<= ctx
['fragment_index']:
386 name
= 'Seg%d-Frag%d' % (seg_i
, frag_i
)
388 if base_url_parsed
.query
:
389 query
.append(base_url_parsed
.query
)
391 query
.append(akamai_pv
.strip(';'))
392 if info_dict
.get('extra_param_to_segment_url'):
393 query
.append(info_dict
['extra_param_to_segment_url'])
394 url_parsed
= base_url_parsed
._replace
(path
=base_url_parsed
.path
+ name
, query
='&'.join(query
))
396 success
, down_data
= self
._download
_fragment
(ctx
, url_parsed
.geturl(), info_dict
)
399 reader
= FlvReader(down_data
)
402 _
, box_type
, box_data
= reader
.read_box_info()
403 except DataTruncatedError
:
405 # In tests, segments may be truncated, and thus
406 # FlvReader may not be able to parse the whole
407 # chunk. If so, write the segment as is
408 # See https://github.com/ytdl-org/youtube-dl/issues/9214
409 dest_stream
.write(down_data
)
412 if box_type
== b
'mdat':
413 self
._append
_fragment
(ctx
, box_data
)
415 except (compat_urllib_error
.HTTPError
, ) as err
:
416 if live
and (err
.code
== 404 or err
.code
== 410):
417 # We didn't keep up with the live window. Continue
418 # with the next available fragment.
419 msg
= 'Fragment %d unavailable' % frag_i
420 self
.report_warning(msg
)
425 if not fragments_list
and not test
and live
and bootstrap_url
:
426 fragments_list
= self
._update
_live
_fragments
(bootstrap_url
, frag_i
)
427 total_frags
+= len(fragments_list
)
428 if fragments_list
and (fragments_list
[0][1] > frag_i
+ 1):
429 msg
= 'Missed %d fragments' % (fragments_list
[0][1] - (frag_i
+ 1))
430 self
.report_warning(msg
)
432 self
._finish
_frag
_download
(ctx
, info_dict
)