1 # -*- coding: utf-8 -*-
2 from __future__
import unicode_literals
11 from .common
import InfoExtractor
12 from ..compat
import (
14 compat_urllib_parse_urlparse
,
27 default_ns
= 'http://www.w3.org/2005/SMIL21/Language'
28 _x
= lambda p
: xpath_with_ns(p
, {'smil': default_ns}
)
31 class ThePlatformBaseIE(InfoExtractor
):
32 def _extract_theplatform_smil(self
, smil_url
, video_id
, note
='Downloading SMIL data'):
33 meta
= self
._download
_xml
(smil_url
, video_id
, note
=note
)
37 for n
in meta
.findall(_x('.//smil:ref'))
38 if n
.attrib
.get('title') == 'Geographic Restriction' or n
.attrib
.get('title') == 'Expired')
42 raise ExtractorError(error_msg
, expected
=True)
44 formats
= self
._parse
_smil
_formats
(
45 meta
, smil_url
, video_id
, namespace
=default_ns
,
46 # the parameters are from syfy.com, other sites may use others,
47 # they also work for nbc.com
48 f4m_params
={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'}
,
49 transform_rtmp_url
=lambda streamer
, src
: (streamer
, 'mp4:' + src
))
51 for _format
in formats
:
52 ext
= determine_ext(_format
['url'])
54 _format
['ext'] = 'mp4'
56 self
._sort
_formats
(formats
)
58 subtitles
= self
._parse
_smil
_subtitles
(meta
, default_ns
)
60 return formats
, subtitles
62 def get_metadata(self
, path
, video_id
):
63 info_url
= 'http://link.theplatform.com/s/%s?format=preview' % path
64 info
= self
._download
_json
(info_url
, video_id
)
67 captions
= info
.get('captions')
68 if isinstance(captions
, list):
69 for caption
in captions
:
70 lang
, src
, mime
= caption
.get('lang', 'en'), caption
.get('src'), caption
.get('type')
72 'ext': 'srt' if mime
== 'text/srt' else 'ttml',
77 'title': info
['title'],
78 'subtitles': subtitles
,
79 'description': info
['description'],
80 'thumbnail': info
['defaultThumbnailUrl'],
81 'duration': int_or_none(info
.get('duration'), 1000),
85 class ThePlatformIE(ThePlatformBaseIE
):
87 (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
88 (?:(?P<media>(?:[^/]+/)+select/media/)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
89 |theplatform:)(?P<id>[^/\?&]+)'''
92 # from http://www.metacafe.com/watch/cb-e9I_cZgTgIPd/blackberrys_big_bold_z30/
93 'url': 'http://link.theplatform.com/s/dJ5BDC/e9I_cZgTgIPd/meta.smil?format=smil&Tracking=true&mbr=true',
97 'title': 'Blackberry\'s big, bold Z30',
98 'description': 'The Z30 is Blackberry\'s biggest, baddest mobile messaging device yet.',
103 'skip_download': True,
106 # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
107 'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',
109 'id': '22d_qsQ6MIRT',
111 'description': 'md5:ac330c9258c04f9d7512cf26b9595409',
112 'title': 'Tesla Model S: A second step towards a cleaner motoring future',
116 'skip_download': True,
119 'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
121 'id': 'yMBg9E8KFxZD',
123 'description': 'md5:644ad9188d655b742f942bf2e06b002d',
124 'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
127 'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
128 'only_matching': True,
130 'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701',
131 'md5': '734f3790fb5fc4903da391beeebc4836',
133 'id': 'tdy_or_siri_150701',
135 'title': 'iPhone Siri’s sassy response to a math question has people talking',
136 'description': 'md5:a565d1deadd5086f3331d57298ec6333',
138 'thumbnail': 're:^https?://.*\.jpg$',
139 'timestamp': 1435752600,
140 'upload_date': '20150701',
141 'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
144 # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
145 # geo-restricted (US), HLS encrypted with AES-128
146 'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
147 'only_matching': True,
151 def _sign_url(url
, sig_key
, sig_secret
, life
=600, include_qs
=False):
152 flags
= '10' if include_qs
else '00'
153 expiration_date
= '%x' % (int(time
.time()) + life
)
156 return binascii
.b2a_hex(str.encode('ascii')).decode('ascii')
159 return binascii
.a2b_hex(hex)
161 relative_path
= url
.split('http://link.theplatform.com/s/')[1].split('?')[0]
162 clear_text
= hex_to_str(flags
+ expiration_date
+ str_to_hex(relative_path
))
163 checksum
= hmac
.new(sig_key
.encode('ascii'), clear_text
, hashlib
.sha1
).hexdigest()
164 sig
= flags
+ expiration_date
+ checksum
+ str_to_hex(sig_secret
)
165 return '%s&sig=%s' % (url
, sig
)
167 def _real_extract(self
, url
):
168 url
, smuggled_data
= unsmuggle_url(url
, {})
170 mobj
= re
.match(self
._VALID
_URL
, url
)
171 provider_id
= mobj
.group('provider_id')
172 video_id
= mobj
.group('id')
175 provider_id
= 'dJ5BDC'
178 if mobj
.group('media'):
180 path
+= '/' + video_id
182 qs_dict
= compat_parse_qs(compat_urllib_parse_urlparse(url
).query
)
183 if 'guid' in qs_dict
:
184 webpage
= self
._download
_webpage
(url
, video_id
)
185 scripts
= re
.findall(r
'<script[^>]+src="([^"]+)"', webpage
)
187 # feed id usually locates in the last script.
188 # Seems there's no pattern for the interested script filename, so
190 for script
in reversed(scripts
):
191 feed_script
= self
._download
_webpage
(
192 self
._proto
_relative
_url
(script
, 'http:'),
193 video_id
, 'Downloading feed script')
194 feed_id
= self
._search
_regex
(
195 r
'defaultFeedId\s*:\s*"([^"]+)"', feed_script
,
196 'default feed id', default
=None)
197 if feed_id
is not None:
200 raise ExtractorError('Unable to find feed id')
201 return self
.url_result('http://feed.theplatform.com/f/%s/%s?byGuid=%s' % (
202 provider_id
, feed_id
, qs_dict
['guid'][0]))
204 if smuggled_data
.get('force_smil_url', False):
206 # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
207 elif '/guid/' in url
:
209 source_url
= smuggled_data
.get('source_url')
211 headers
['Referer'] = source_url
212 request
= sanitized_Request(url
, headers
=headers
)
213 webpage
= self
._download
_webpage
(request
, video_id
)
214 smil_url
= self
._search
_regex
(
215 r
'<link[^>]+href=(["\'])(?P
<url
>.+?
)\
1[^
>]+type=["\']application/smil\+xml',
216 webpage, 'smil url', group='url')
217 path = self._search_regex(
218 r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
219 smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4&format=SMIL'
220 elif mobj.group('config'):
221 config_url = url + '&form=json'
222 config_url = config_url.replace('swf/', 'config/')
223 config_url = config_url.replace('onsite/', 'onsite/config/')
224 config = self._download_json(config_url, video_id, 'Downloading config')
225 if 'releaseUrl' in config:
226 release_url = config['releaseUrl']
228 release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
229 smil_url = release_url + '&format=SMIL&formats=MPEG4&manifest=f4m'
231 smil_url = 'http://link.theplatform.com/s/%s/meta.smil?format=smil&mbr=true' % path
233 sig = smuggled_data.get('sig')
235 smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
237 formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
239 ret = self.get_metadata(path, video_id)
240 combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
244 'subtitles': combined_subtitles,
250 class ThePlatformFeedIE(ThePlatformBaseIE):
251 _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&byGuid=%s'
252 _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P<provider_id>[^/]+)/(?P<feed_id>[^?/]+)\?(?:[^&]+&)*byGuid=(?P<id>[a-zA-Z0-9_]+)'
254 # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
255 'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
256 'md5': '22d2b84f058d3586efcd99e57d59d314',
258 'id': 'n_hardball_5biden_140207',
260 'title': 'The Biden factor: will Joe run in 2016?',
261 'description': 'Could Vice President Joe Biden be preparing a 2016 campaign? Mark Halperin and Sam Stein weigh in.',
262 'thumbnail': 're:^https?://.*\.jpg$',
263 'upload_date': '20140208',
264 'timestamp': 1391824260,
266 'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
270 def _real_extract(self, url):
271 mobj = re.match(self._VALID_URL, url)
273 video_id = mobj.group('id')
274 provider_id = mobj.group('provider_id')
275 feed_id = mobj.group('feed_id')
277 real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, video_id)
278 feed = self._download_json(real_url, video_id)
279 entry = feed['entries'][0]
283 first_video_id = None
285 for item in entry['media$content']:
286 smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M'
287 cur_video_id = url_basename(smil_url)
288 if first_video_id is None:
289 first_video_id = cur_video_id
290 duration = float_or_none(item.get('plfile$duration'))
291 cur_formats, cur_subtitles = self._extract_theplatform_smil(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id)
292 formats.extend(cur_formats)
293 subtitles = self._merge_subtitles(subtitles, cur_subtitles)
295 self._sort_formats(formats)
298 'url': thumbnail['plfile$url'],
299 'width': int_or_none(thumbnail.get('plfile$width')),
300 'height': int_or_none(thumbnail.get('plfile$height')),
301 } for thumbnail in entry.get('media$thumbnails', [])]
303 timestamp = int_or_none(entry.get('media$availableDate'), scale=1000)
304 categories = [item['media$name'] for item in entry.get('media$categories', [])]
306 ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id)
307 subtitles = self._merge_subtitles(subtitles, ret['subtitles'])
311 'subtitles': subtitles,
312 'thumbnails': thumbnails,
313 'duration': duration,
314 'timestamp': timestamp,
315 'categories': categories,