]>
jfr.im git - yt-dlp.git/blob - youtube_dl/downloader/f4m.py
1 from __future__
import division
, unicode_literals
8 import xml
.etree
.ElementTree
as etree
10 from .common
import FileDownloader
11 from .http
import HttpFD
12 from ..compat
import (
25 class FlvReader(io
.BytesIO
):
28 The file format is documented in https://www.adobe.com/devnet/f4v.html
31 # Utility functions for reading numbers and strings
32 def read_unsigned_long_long(self
):
33 return struct_unpack('!Q', self
.read(8))[0]
35 def read_unsigned_int(self
):
36 return struct_unpack('!I', self
.read(4))[0]
38 def read_unsigned_char(self
):
39 return struct_unpack('!B', self
.read(1))[0]
41 def read_string(self
):
50 def read_box_info(self
):
52 Read a box and return the info as a tuple: (box_size, box_type, box_data)
54 real_size
= size
= self
.read_unsigned_int()
55 box_type
= self
.read(4)
58 real_size
= self
.read_unsigned_long_long()
60 return real_size
, box_type
, self
.read(real_size
- header_end
)
64 self
.read_unsigned_char()
67 quality_entry_count
= self
.read_unsigned_char()
69 for i
in range(quality_entry_count
):
72 segment_run_count
= self
.read_unsigned_int()
74 for i
in range(segment_run_count
):
75 first_segment
= self
.read_unsigned_int()
76 fragments_per_segment
= self
.read_unsigned_int()
77 segments
.append((first_segment
, fragments_per_segment
))
80 'segment_run': segments
,
85 self
.read_unsigned_char()
89 self
.read_unsigned_int()
91 quality_entry_count
= self
.read_unsigned_char()
92 # QualitySegmentUrlModifiers
93 for i
in range(quality_entry_count
):
96 fragments_count
= self
.read_unsigned_int()
98 for i
in range(fragments_count
):
99 first
= self
.read_unsigned_int()
100 first_ts
= self
.read_unsigned_long_long()
101 duration
= self
.read_unsigned_int()
103 discontinuity_indicator
= self
.read_unsigned_char()
105 discontinuity_indicator
= None
109 'duration': duration
,
110 'discontinuity_indicator': discontinuity_indicator
,
114 'fragments': fragments
,
119 self
.read_unsigned_char()
123 self
.read_unsigned_int() # BootstrapinfoVersion
124 # Profile,Live,Update,Reserved
125 flags
= self
.read_unsigned_char()
126 live
= flags
& 0x20 != 0
128 self
.read_unsigned_int()
130 self
.read_unsigned_long_long()
131 # SmpteTimeCodeOffset
132 self
.read_unsigned_long_long()
134 self
.read_string() # MovieIdentifier
135 server_count
= self
.read_unsigned_char()
137 for i
in range(server_count
):
139 quality_count
= self
.read_unsigned_char()
141 for i
in range(quality_count
):
148 segments_count
= self
.read_unsigned_char()
150 for i
in range(segments_count
):
151 box_size
, box_type
, box_data
= self
.read_box_info()
152 assert box_type
== b
'asrt'
153 segment
= FlvReader(box_data
).read_asrt()
154 segments
.append(segment
)
155 fragments_run_count
= self
.read_unsigned_char()
157 for i
in range(fragments_run_count
):
158 box_size
, box_type
, box_data
= self
.read_box_info()
159 assert box_type
== b
'afrt'
160 fragments
.append(FlvReader(box_data
).read_afrt())
163 'segments': segments
,
164 'fragments': fragments
,
168 def read_bootstrap_info(self
):
169 total_size
, box_type
, box_data
= self
.read_box_info()
170 assert box_type
== b
'abst'
171 return FlvReader(box_data
).read_abst()
174 def read_bootstrap_info(bootstrap_bytes
):
175 return FlvReader(bootstrap_bytes
).read_bootstrap_info()
178 def build_fragments_list(boot_info
):
179 """ Return a list of (segment, fragment) for each fragment in the video """
181 segment_run_table
= boot_info
['segments'][0]
182 fragment_run_entry_table
= boot_info
['fragments'][0]['fragments']
183 first_frag_number
= fragment_run_entry_table
[0]['first']
184 fragments_counter
= itertools
.count(first_frag_number
)
185 for segment
, fragments_count
in segment_run_table
['segment_run']:
186 for _
in range(fragments_count
):
187 res
.append((segment
, next(fragments_counter
)))
189 if boot_info
['live']:
195 def write_unsigned_int(stream
, val
):
196 stream
.write(struct_pack('!I', val
))
199 def write_unsigned_int_24(stream
, val
):
200 stream
.write(struct_pack('!I', val
)[1:])
203 def write_flv_header(stream
):
204 """Writes the FLV header to stream"""
206 stream
.write(b
'FLV\x01')
207 stream
.write(b
'\x05')
208 stream
.write(b
'\x00\x00\x00\x09')
209 stream
.write(b
'\x00\x00\x00\x00')
212 def write_metadata_tag(stream
, metadata
):
213 """Writes optional metadata tag to stream"""
215 FLV_TAG_HEADER_LEN
= 11
218 stream
.write(SCRIPT_TAG
)
219 write_unsigned_int_24(stream
, len(metadata
))
220 stream
.write(b
'\x00\x00\x00\x00\x00\x00\x00')
221 stream
.write(metadata
)
222 write_unsigned_int(stream
, FLV_TAG_HEADER_LEN
+ len(metadata
))
226 return '{http://ns.adobe.com/f4m/1.0}%s' % prop
229 class HttpQuietDownloader(HttpFD
):
230 def to_screen(self
, *args
, **kargs
):
234 class F4mFD(FileDownloader
):
236 A downloader for f4m manifests or AdobeHDS.
239 def _get_unencrypted_media(self
, doc
):
240 media
= doc
.findall(_add_ns('media'))
242 self
.report_error('No media found')
243 for e
in (doc
.findall(_add_ns('drmAdditionalHeader')) +
244 doc
.findall(_add_ns('drmAdditionalHeaderSet'))):
245 # If id attribute is missing it's valid for all media nodes
246 # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
247 if 'id' not in e
.attrib
:
248 self
.report_error('Missing ID in f4m DRM')
249 media
= list(filter(lambda e
: 'drmAdditionalHeaderId' not in e
.attrib
and
250 'drmAdditionalHeaderSetId' not in e
.attrib
,
253 self
.report_error('Unsupported DRM')
256 def _get_bootstrap_from_url(self
, bootstrap_url
):
257 bootstrap
= self
.ydl
.urlopen(bootstrap_url
).read()
258 return read_bootstrap_info(bootstrap
)
260 def _update_live_fragments(self
, bootstrap_url
, latest_fragment
):
263 while (not fragments_list
) and (retries
> 0):
264 boot_info
= self
._get
_bootstrap
_from
_url
(bootstrap_url
)
265 fragments_list
= build_fragments_list(boot_info
)
266 fragments_list
= [f
for f
in fragments_list
if f
[1] > latest_fragment
]
267 if not fragments_list
:
268 # Retry after a while
272 if not fragments_list
:
273 self
.report_error('Failed to update fragments')
275 return fragments_list
277 def _parse_bootstrap_node(self
, node
, base_url
):
278 if node
.text
is None:
279 bootstrap_url
= compat_urlparse
.urljoin(
280 base_url
, node
.attrib
['url'])
281 boot_info
= self
._get
_bootstrap
_from
_url
(bootstrap_url
)
284 bootstrap
= base64
.b64decode(node
.text
)
285 boot_info
= read_bootstrap_info(bootstrap
)
286 return (boot_info
, bootstrap_url
)
288 def real_download(self
, filename
, info_dict
):
289 man_url
= info_dict
['url']
290 requested_bitrate
= info_dict
.get('tbr')
291 self
.to_screen('[download] Downloading f4m manifest')
292 manifest
= self
.ydl
.urlopen(man_url
).read()
294 doc
= etree
.fromstring(manifest
)
295 formats
= [(int(f
.attrib
.get('bitrate', -1)), f
)
296 for f
in self
._get
_unencrypted
_media
(doc
)]
297 if requested_bitrate
is None:
298 # get the best format
299 formats
= sorted(formats
, key
=lambda f
: f
[0])
300 rate
, media
= formats
[-1]
302 rate
, media
= list(filter(
303 lambda f
: int(f
[0]) == requested_bitrate
, formats
))[0]
305 base_url
= compat_urlparse
.urljoin(man_url
, media
.attrib
['url'])
306 bootstrap_node
= doc
.find(_add_ns('bootstrapInfo'))
307 boot_info
, bootstrap_url
= self
._parse
_bootstrap
_node
(bootstrap_node
, base_url
)
308 live
= boot_info
['live']
309 metadata_node
= media
.find(_add_ns('metadata'))
310 if metadata_node
is not None:
311 metadata
= base64
.b64decode(metadata_node
.text
)
315 fragments_list
= build_fragments_list(boot_info
)
316 if self
.params
.get('test', False):
317 # We only download the first fragment
318 fragments_list
= fragments_list
[:1]
319 total_frags
= len(fragments_list
)
320 # For some akamai manifests we'll need to add a query to the fragment url
321 akamai_pv
= xpath_text(doc
, _add_ns('pv-2.0'))
323 self
.report_destination(filename
)
324 http_dl
= HttpQuietDownloader(
330 'ratelimit': self
.params
.get('ratelimit', None),
331 'test': self
.params
.get('test', False),
334 tmpfilename
= self
.temp_name(filename
)
335 (dest_stream
, tmpfilename
) = sanitize_open(tmpfilename
, 'wb')
337 write_flv_header(dest_stream
)
339 write_metadata_tag(dest_stream
, metadata
)
341 # This dict stores the download progress, it's updated by the progress
344 'status': 'downloading',
345 'downloaded_bytes': 0,
347 'frag_count': total_frags
,
348 'filename': filename
,
349 'tmpfilename': tmpfilename
,
353 def frag_progress_hook(s
):
354 if s
['status'] not in ('downloading', 'finished'):
357 frag_total_bytes
= s
.get('total_bytes', 0)
358 if s
['status'] == 'finished':
359 state
['downloaded_bytes'] += frag_total_bytes
360 state
['frag_index'] += 1
363 (state
['downloaded_bytes'] + frag_total_bytes
) /
364 (state
['frag_index'] + 1) * total_frags
)
365 time_now
= time
.time()
366 state
['total_bytes_estimate'] = estimated_size
367 state
['elapsed'] = time_now
- start
369 if s
['status'] == 'finished':
370 progress
= self
.calc_percent(state
['frag_index'], total_frags
)
372 frag_downloaded_bytes
= s
['downloaded_bytes']
373 frag_progress
= self
.calc_percent(frag_downloaded_bytes
,
375 progress
= self
.calc_percent(state
['frag_index'], total_frags
)
376 progress
+= frag_progress
/ float(total_frags
)
378 state
['eta'] = self
.calc_eta(
379 start
, time_now
, estimated_size
, state
['downloaded_bytes'] + frag_downloaded_bytes
)
380 state
['speed'] = s
.get('speed')
381 self
._hook
_progress
(state
)
383 http_dl
.add_progress_hook(frag_progress_hook
)
386 while fragments_list
:
387 seg_i
, frag_i
= fragments_list
.pop(0)
388 name
= 'Seg%d-Frag%d' % (seg_i
, frag_i
)
389 url
= base_url
+ name
391 url
+= '?' + akamai_pv
.strip(';')
392 frag_filename
= '%s-%s' % (tmpfilename
, name
)
394 success
= http_dl
.download(frag_filename
, {'url': url}
)
397 with open(frag_filename
, 'rb') as down
:
398 down_data
= down
.read()
399 reader
= FlvReader(down_data
)
401 _
, box_type
, box_data
= reader
.read_box_info()
402 if box_type
== b
'mdat':
403 dest_stream
.write(box_data
)
406 os
.remove(frag_filename
)
408 frags_filenames
.append(frag_filename
)
409 except (compat_urllib_error
.HTTPError
, ) as err
:
410 if live
and (err
.code
== 404 or err
.code
== 410):
411 # We didn't keep up with the live window. Continue
412 # with the next available fragment.
413 msg
= 'Fragment %d unavailable' % frag_i
414 self
.report_warning(msg
)
419 if not fragments_list
and live
and bootstrap_url
:
420 fragments_list
= self
._update
_live
_fragments
(bootstrap_url
, frag_i
)
421 total_frags
+= len(fragments_list
)
422 if fragments_list
and (fragments_list
[0][1] > frag_i
+ 1):
423 msg
= 'Missed %d fragments' % (fragments_list
[0][1] - (frag_i
+ 1))
424 self
.report_warning(msg
)
428 elapsed
= time
.time() - start
429 self
.try_rename(tmpfilename
, filename
)
430 for frag_file
in frags_filenames
:
433 fsize
= os
.path
.getsize(encodeFilename(filename
))
434 self
._hook
_progress
({
435 'downloaded_bytes': fsize
,
436 'total_bytes': fsize
,
437 'filename': filename
,
438 'status': 'finished',