]> jfr.im git - yt-dlp.git/blob - yt_dlp/extractor/espn.py
[extractor/ESPN] Extract duration (#4499)
[yt-dlp.git] / yt_dlp / extractor / espn.py
1 import base64
2 import json
3 import re
4 import urllib.parse
5
6 from .adobepass import AdobePassIE
7 from .common import InfoExtractor
8 from .once import OnceIE
9 from ..utils import (
10 determine_ext,
11 dict_get,
12 int_or_none,
13 traverse_obj,
14 unified_strdate,
15 unified_timestamp,
16 )
17
18
19 class ESPNIE(OnceIE):
20 _VALID_URL = r'''(?x)
21 https?://
22 (?:
23 (?:
24 (?:
25 (?:(?:\w+\.)+)?espn\.go|
26 (?:www\.)?espn
27 )\.com/
28 (?:
29 (?:
30 video/(?:clip|iframe/twitter)|
31 )
32 (?:
33 .*?\?.*?\bid=|
34 /_/id/
35 )|
36 [^/]+/video/
37 )
38 )|
39 (?:www\.)espnfc\.(?:com|us)/(?:video/)?[^/]+/\d+/video/
40 )
41 (?P<id>\d+)
42 '''
43
44 _TESTS = [{
45 'url': 'http://espn.go.com/video/clip?id=10365079',
46 'info_dict': {
47 'id': '10365079',
48 'ext': 'mp4',
49 'title': '30 for 30 Shorts: Judging Jewell',
50 'description': 'md5:39370c2e016cb4ecf498ffe75bef7f0f',
51 'timestamp': 1390936111,
52 'upload_date': '20140128',
53 'duration': 1302,
54 'thumbnail': r're:https://.+\.jpg',
55 },
56 'params': {
57 'skip_download': True,
58 },
59 }, {
60 'url': 'https://broadband.espn.go.com/video/clip?id=18910086',
61 'info_dict': {
62 'id': '18910086',
63 'ext': 'mp4',
64 'title': 'Kyrie spins around defender for two',
65 'description': 'md5:2b0f5bae9616d26fba8808350f0d2b9b',
66 'timestamp': 1489539155,
67 'upload_date': '20170315',
68 },
69 'params': {
70 'skip_download': True,
71 },
72 'expected_warnings': ['Unable to download f4m manifest'],
73 }, {
74 'url': 'http://nonredline.sports.espn.go.com/video/clip?id=19744672',
75 'only_matching': True,
76 }, {
77 'url': 'https://cdn.espn.go.com/video/clip/_/id/19771774',
78 'only_matching': True,
79 }, {
80 'url': 'http://www.espn.com/video/clip?id=10365079',
81 'only_matching': True,
82 }, {
83 'url': 'http://www.espn.com/video/clip/_/id/17989860',
84 'only_matching': True,
85 }, {
86 'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
87 'only_matching': True,
88 }, {
89 'url': 'http://www.espnfc.us/video/espn-fc-tv/86/video/3319154/nashville-unveiled-as-the-newest-club-in-mls',
90 'only_matching': True,
91 }, {
92 'url': 'http://www.espnfc.com/english-premier-league/23/video/3324163/premier-league-in-90-seconds-golden-tweets',
93 'only_matching': True,
94 }, {
95 'url': 'http://www.espn.com/espnw/video/26066627/arkansas-gibson-completes-hr-cycle-four-innings',
96 'only_matching': True,
97 }, {
98 'url': 'http://www.espn.com/watch/player?id=19141491',
99 'only_matching': True,
100 }, {
101 'url': 'http://www.espn.com/watch/player?bucketId=257&id=19505875',
102 'only_matching': True,
103 }, ]
104
105 def _real_extract(self, url):
106 video_id = self._match_id(url)
107
108 clip = self._download_json(
109 'http://api-app.espn.com/v1/video/clips/%s' % video_id,
110 video_id)['videos'][0]
111
112 title = clip['headline']
113
114 format_urls = set()
115 formats = []
116
117 def traverse_source(source, base_source_id=None):
118 for source_id, source in source.items():
119 if source_id == 'alert':
120 continue
121 elif isinstance(source, str):
122 extract_source(source, base_source_id)
123 elif isinstance(source, dict):
124 traverse_source(
125 source,
126 '%s-%s' % (base_source_id, source_id)
127 if base_source_id else source_id)
128
129 def extract_source(source_url, source_id=None):
130 if source_url in format_urls:
131 return
132 format_urls.add(source_url)
133 ext = determine_ext(source_url)
134 if OnceIE.suitable(source_url):
135 formats.extend(self._extract_once_formats(source_url))
136 elif ext == 'smil':
137 formats.extend(self._extract_smil_formats(
138 source_url, video_id, fatal=False))
139 elif ext == 'f4m':
140 formats.extend(self._extract_f4m_formats(
141 source_url, video_id, f4m_id=source_id, fatal=False))
142 elif ext == 'm3u8':
143 formats.extend(self._extract_m3u8_formats(
144 source_url, video_id, 'mp4', entry_protocol='m3u8_native',
145 m3u8_id=source_id, fatal=False))
146 else:
147 f = {
148 'url': source_url,
149 'format_id': source_id,
150 }
151 mobj = re.search(r'(\d+)p(\d+)_(\d+)k\.', source_url)
152 if mobj:
153 f.update({
154 'height': int(mobj.group(1)),
155 'fps': int(mobj.group(2)),
156 'tbr': int(mobj.group(3)),
157 })
158 if source_id == 'mezzanine':
159 f['quality'] = 1
160 formats.append(f)
161
162 links = clip.get('links', {})
163 traverse_source(links.get('source', {}))
164 traverse_source(links.get('mobile', {}))
165 self._sort_formats(formats)
166
167 description = clip.get('caption') or clip.get('description')
168 thumbnail = clip.get('thumbnail')
169 duration = int_or_none(clip.get('duration'))
170 timestamp = unified_timestamp(clip.get('originalPublishDate'))
171
172 return {
173 'id': video_id,
174 'title': title,
175 'description': description,
176 'thumbnail': thumbnail,
177 'timestamp': timestamp,
178 'duration': duration,
179 'formats': formats,
180 }
181
182
183 class ESPNArticleIE(InfoExtractor):
184 _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/(?:[^/]+/)*(?P<id>[^/]+)'
185 _TESTS = [{
186 'url': 'http://espn.go.com/nba/recap?gameId=400793786',
187 'only_matching': True,
188 }, {
189 'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
190 'only_matching': True,
191 }, {
192 'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
193 'only_matching': True,
194 }, {
195 'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
196 'only_matching': True,
197 }]
198
199 @classmethod
200 def suitable(cls, url):
201 return False if (ESPNIE.suitable(url) or WatchESPNIE.suitable(url)) else super().suitable(url)
202
203 def _real_extract(self, url):
204 video_id = self._match_id(url)
205
206 webpage = self._download_webpage(url, video_id)
207
208 video_id = self._search_regex(
209 r'class=(["\']).*?video-play-button.*?\1[^>]+data-id=["\'](?P<id>\d+)',
210 webpage, 'video id', group='id')
211
212 return self.url_result(
213 'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
214
215
216 class FiveThirtyEightIE(InfoExtractor):
217 _VALID_URL = r'https?://(?:www\.)?fivethirtyeight\.com/features/(?P<id>[^/?#]+)'
218 _TEST = {
219 'url': 'http://fivethirtyeight.com/features/how-the-6-8-raiders-can-still-make-the-playoffs/',
220 'info_dict': {
221 'id': '56032156',
222 'ext': 'flv',
223 'title': 'FiveThirtyEight: The Raiders can still make the playoffs',
224 'description': 'Neil Paine breaks down the simplest scenario that will put the Raiders into the playoffs at 8-8.',
225 },
226 'params': {
227 'skip_download': True,
228 },
229 }
230
231 def _real_extract(self, url):
232 video_id = self._match_id(url)
233
234 webpage = self._download_webpage(url, video_id)
235
236 embed_url = self._search_regex(
237 r'<iframe[^>]+src=["\'](https?://fivethirtyeight\.abcnews\.go\.com/video/embed/\d+/\d+)',
238 webpage, 'embed url')
239
240 return self.url_result(embed_url, 'AbcNewsVideo')
241
242
243 class ESPNCricInfoIE(InfoExtractor):
244 _VALID_URL = r'https?://(?:www\.)?espncricinfo\.com/video/[^#$&?/]+-(?P<id>\d+)'
245 _TESTS = [{
246 'url': 'https://www.espncricinfo.com/video/finch-chasing-comes-with-risks-despite-world-cup-trend-1289135',
247 'info_dict': {
248 'id': '1289135',
249 'ext': 'mp4',
250 'title': 'Finch: Chasing comes with \'risks\' despite World Cup trend',
251 'description': 'md5:ea32373303e25efbb146efdfc8a37829',
252 'upload_date': '20211113',
253 'duration': 96,
254 },
255 'params': {'skip_download': True}
256 }]
257
258 def _real_extract(self, url):
259 id = self._match_id(url)
260 data_json = self._download_json(f'https://hs-consumer-api.espncricinfo.com/v1/pages/video/video-details?videoId={id}', id)['video']
261 formats, subtitles = [], {}
262 for item in data_json.get('playbacks') or []:
263 if item.get('type') == 'HLS' and item.get('url'):
264 m3u8_frmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(item['url'], id)
265 formats.extend(m3u8_frmts)
266 subtitles = self._merge_subtitles(subtitles, m3u8_subs)
267 elif item.get('type') == 'AUDIO' and item.get('url'):
268 formats.append({
269 'url': item['url'],
270 'vcodec': 'none',
271 })
272 self._sort_formats(formats)
273 return {
274 'id': id,
275 'title': data_json.get('title'),
276 'description': data_json.get('summary'),
277 'upload_date': unified_strdate(dict_get(data_json, ('publishedAt', 'recordedAt'))),
278 'duration': data_json.get('duration'),
279 'formats': formats,
280 'subtitles': subtitles,
281 }
282
283
284 class WatchESPNIE(AdobePassIE):
285 _VALID_URL = r'https?://(?:www\.)?espn\.com/(?:watch|espnplus)/player/_/id/(?P<id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'
286 _TESTS = [{
287 'url': 'https://www.espn.com/watch/player/_/id/dbbc6b1d-c084-4b47-9878-5f13c56ce309',
288 'info_dict': {
289 'id': 'dbbc6b1d-c084-4b47-9878-5f13c56ce309',
290 'ext': 'mp4',
291 'title': 'Huddersfield vs. Burnley',
292 'duration': 7500,
293 'thumbnail': 'https://artwork.api.espn.com/artwork/collections/media/dbbc6b1d-c084-4b47-9878-5f13c56ce309/default?width=640&apikey=1ngjw23osgcis1i1vbj96lmfqs',
294 },
295 'params': {
296 'skip_download': True,
297 },
298 }, {
299 'url': 'https://www.espn.com/watch/player/_/id/a049a56e-a7ce-477e-aef3-c7e48ef8221c',
300 'info_dict': {
301 'id': 'a049a56e-a7ce-477e-aef3-c7e48ef8221c',
302 'ext': 'mp4',
303 'title': 'Dynamo Dresden vs. VfB Stuttgart (Round #1) (German Cup)',
304 'duration': 8335,
305 'thumbnail': 'https://s.secure.espncdn.com/stitcher/artwork/collections/media/bd1f3d12-0654-47d9-852e-71b85ea695c7/16x9.jpg?timestamp=202201112217&showBadge=true&cb=12&package=ESPN_PLUS',
306 },
307 'params': {
308 'skip_download': True,
309 },
310 }, {
311 'url': 'https://www.espn.com/espnplus/player/_/id/317f5fd1-c78a-4ebe-824a-129e0d348421',
312 'info_dict': {
313 'id': '317f5fd1-c78a-4ebe-824a-129e0d348421',
314 'ext': 'mp4',
315 'title': 'The Wheel - Episode 10',
316 'duration': 3352,
317 'thumbnail': 'https://s.secure.espncdn.com/stitcher/artwork/collections/media/317f5fd1-c78a-4ebe-824a-129e0d348421/16x9.jpg?timestamp=202205031523&showBadge=true&cb=12&package=ESPN_PLUS',
318 },
319 'params': {
320 'skip_download': True,
321 },
322 }]
323
324 _API_KEY = 'ZXNwbiZicm93c2VyJjEuMC4w.ptUt7QxsteaRruuPmGZFaJByOoqKvDP2a5YkInHrc7c'
325
326 def _call_bamgrid_api(self, path, video_id, payload=None, headers={}):
327 if 'Authorization' not in headers:
328 headers['Authorization'] = f'Bearer {self._API_KEY}'
329 parse = urllib.parse.urlencode if path == 'token' else json.dumps
330 return self._download_json(
331 f'https://espn.api.edge.bamgrid.com/{path}', video_id, headers=headers, data=parse(payload).encode())
332
333 def _real_extract(self, url):
334 video_id = self._match_id(url)
335 cdn_data = self._download_json(
336 f'https://watch-cdn.product.api.espn.com/api/product/v3/watchespn/web/playback/event?id={video_id}',
337 video_id)
338 video_data = cdn_data['playbackState']
339
340 # ESPN+ subscription required, through cookies
341 if 'DTC' in video_data.get('sourceId'):
342 cookie = self._get_cookies(url).get('ESPN-ONESITE.WEB-PROD.token')
343 if not cookie:
344 self.raise_login_required(method='cookies')
345
346 assertion = self._call_bamgrid_api(
347 'devices', video_id,
348 headers={'Content-Type': 'application/json; charset=UTF-8'},
349 payload={
350 'deviceFamily': 'android',
351 'applicationRuntime': 'android',
352 'deviceProfile': 'tv',
353 'attributes': {},
354 })['assertion']
355 token = self._call_bamgrid_api(
356 'token', video_id, payload={
357 'subject_token': assertion,
358 'subject_token_type': 'urn:bamtech:params:oauth:token-type:device',
359 'platform': 'android',
360 'grant_type': 'urn:ietf:params:oauth:grant-type:token-exchange'
361 })['access_token']
362
363 assertion = self._call_bamgrid_api(
364 'accounts/grant', video_id, payload={'id_token': cookie.value.split('|')[1]},
365 headers={
366 'Authorization': token,
367 'Content-Type': 'application/json; charset=UTF-8'
368 })['assertion']
369 token = self._call_bamgrid_api(
370 'token', video_id, payload={
371 'subject_token': assertion,
372 'subject_token_type': 'urn:bamtech:params:oauth:token-type:account',
373 'platform': 'android',
374 'grant_type': 'urn:ietf:params:oauth:grant-type:token-exchange'
375 })['access_token']
376
377 playback = self._download_json(
378 video_data['videoHref'].format(scenario='browser~ssai'), video_id,
379 headers={
380 'Accept': 'application/vnd.media-service+json; version=5',
381 'Authorization': token
382 })
383 m3u8_url, headers = playback['stream']['complete'][0]['url'], {'authorization': token}
384
385 # No login required
386 elif video_data.get('sourceId') == 'ESPN_FREE':
387 asset = self._download_json(
388 f'https://watch.auth.api.espn.com/video/auth/media/{video_id}/asset?apikey=uiqlbgzdwuru14v627vdusswb',
389 video_id)
390 m3u8_url, headers = asset['stream'], {}
391
392 # TV Provider required
393 else:
394 resource = self._get_mvpd_resource('ESPN', video_data['name'], video_id, None)
395 auth = self._extract_mvpd_auth(url, video_id, 'ESPN', resource).encode()
396
397 asset = self._download_json(
398 f'https://watch.auth.api.espn.com/video/auth/media/{video_id}/asset?apikey=uiqlbgzdwuru14v627vdusswb',
399 video_id, data=f'adobeToken={urllib.parse.quote_plus(base64.b64encode(auth))}&drmSupport=HLS'.encode())
400 m3u8_url, headers = asset['stream'], {}
401
402 formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
403 self._sort_formats(formats)
404
405 return {
406 'id': video_id,
407 'duration': traverse_obj(cdn_data, ('tracking', 'duration')),
408 'title': video_data.get('name'),
409 'formats': formats,
410 'subtitles': subtitles,
411 'thumbnail': video_data.get('posterHref'),
412 'http_headers': headers,
413 }