yt_dlp/extractor/espn.py

   1 from __future__ import unicode_literals
   2
   3 import re
   4
   5 from .common import InfoExtractor
   6 from .once import OnceIE
   7 from ..compat import compat_str
   8 from ..utils import (
   9     determine_ext,
  10     dict_get,
  11     int_or_none,
  12     unified_strdate,
  13     unified_timestamp,
  14 )
  15
  16
  17 class ESPNIE(OnceIE):
  18     _VALID_URL = r'''(?x)
  19                     https?://
  20                         (?:
  21                             (?:
  22                                 (?:
  23                                     (?:(?:\w+\.)+)?espn\.go|
  24                                     (?:www\.)?espn
  25                                 )\.com/
  26                                 (?:
  27                                     (?:
  28                                         video/(?:clip|iframe/twitter)|
  29                                         watch/player
  30                                     )
  31                                     (?:
  32                                         .*?\?.*?\bid=|
  33                                         /_/id/
  34                                     )|
  35                                     [^/]+/video/
  36                                 )
  37                             )|
  38                             (?:www\.)espnfc\.(?:com|us)/(?:video/)?[^/]+/\d+/video/
  39                         )
  40                         (?P<id>\d+)
  41                     '''
  42
  43     _TESTS = [{
  44         'url': 'http://espn.go.com/video/clip?id=10365079',
  45         'info_dict': {
  46             'id': '10365079',
  47             'ext': 'mp4',
  48             'title': '30 for 30 Shorts: Judging Jewell',
  49             'description': 'md5:39370c2e016cb4ecf498ffe75bef7f0f',
  50             'timestamp': 1390936111,
  51             'upload_date': '20140128',
  52         },
  53         'params': {
  54             'skip_download': True,
  55         },
  56     }, {
  57         'url': 'https://broadband.espn.go.com/video/clip?id=18910086',
  58         'info_dict': {
  59             'id': '18910086',
  60             'ext': 'mp4',
  61             'title': 'Kyrie spins around defender for two',
  62             'description': 'md5:2b0f5bae9616d26fba8808350f0d2b9b',
  63             'timestamp': 1489539155,
  64             'upload_date': '20170315',
  65         },
  66         'params': {
  67             'skip_download': True,
  68         },
  69         'expected_warnings': ['Unable to download f4m manifest'],
  70     }, {
  71         'url': 'http://nonredline.sports.espn.go.com/video/clip?id=19744672',
  72         'only_matching': True,
  73     }, {
  74         'url': 'https://cdn.espn.go.com/video/clip/_/id/19771774',
  75         'only_matching': True,
  76     }, {
  77         'url': 'http://www.espn.com/watch/player?id=19141491',
  78         'only_matching': True,
  79     }, {
  80         'url': 'http://www.espn.com/watch/player?bucketId=257&id=19505875',
  81         'only_matching': True,
  82     }, {
  83         'url': 'http://www.espn.com/watch/player/_/id/19141491',
  84         'only_matching': True,
  85     }, {
  86         'url': 'http://www.espn.com/video/clip?id=10365079',
  87         'only_matching': True,
  88     }, {
  89         'url': 'http://www.espn.com/video/clip/_/id/17989860',
  90         'only_matching': True,
  91     }, {
  92         'url': 'https://espn.go.com/video/iframe/twitter/?cms=espn&id=10365079',
  93         'only_matching': True,
  94     }, {
  95         'url': 'http://www.espnfc.us/video/espn-fc-tv/86/video/3319154/nashville-unveiled-as-the-newest-club-in-mls',
  96         'only_matching': True,
  97     }, {
  98         'url': 'http://www.espnfc.com/english-premier-league/23/video/3324163/premier-league-in-90-seconds-golden-tweets',
  99         'only_matching': True,
 100     }, {
 101         'url': 'http://www.espn.com/espnw/video/26066627/arkansas-gibson-completes-hr-cycle-four-innings',
 102         'only_matching': True,
 103     }]
 104
 105     def _real_extract(self, url):
 106         video_id = self._match_id(url)
 107
 108         clip = self._download_json(
 109             'http://api-app.espn.com/v1/video/clips/%s' % video_id,
 110             video_id)['videos'][0]
 111
 112         title = clip['headline']
 113
 114         format_urls = set()
 115         formats = []
 116
 117         def traverse_source(source, base_source_id=None):
 118             for source_id, source in source.items():
 119                 if source_id == 'alert':
 120                     continue
 121                 elif isinstance(source, compat_str):
 122                     extract_source(source, base_source_id)
 123                 elif isinstance(source, dict):
 124                     traverse_source(
 125                         source,
 126                         '%s-%s' % (base_source_id, source_id)
 127                         if base_source_id else source_id)
 128
 129         def extract_source(source_url, source_id=None):
 130             if source_url in format_urls:
 131                 return
 132             format_urls.add(source_url)
 133             ext = determine_ext(source_url)
 134             if OnceIE.suitable(source_url):
 135                 formats.extend(self._extract_once_formats(source_url))
 136             elif ext == 'smil':
 137                 formats.extend(self._extract_smil_formats(
 138                     source_url, video_id, fatal=False))
 139             elif ext == 'f4m':
 140                 formats.extend(self._extract_f4m_formats(
 141                     source_url, video_id, f4m_id=source_id, fatal=False))
 142             elif ext == 'm3u8':
 143                 formats.extend(self._extract_m3u8_formats(
 144                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
 145                     m3u8_id=source_id, fatal=False))
 146             else:
 147                 f = {
 148                     'url': source_url,
 149                     'format_id': source_id,
 150                 }
 151                 mobj = re.search(r'(\d+)p(\d+)_(\d+)k\.', source_url)
 152                 if mobj:
 153                     f.update({
 154                         'height': int(mobj.group(1)),
 155                         'fps': int(mobj.group(2)),
 156                         'tbr': int(mobj.group(3)),
 157                     })
 158                 if source_id == 'mezzanine':
 159                     f['quality'] = 1
 160                 formats.append(f)
 161
 162         links = clip.get('links', {})
 163         traverse_source(links.get('source', {}))
 164         traverse_source(links.get('mobile', {}))
 165         self._sort_formats(formats)
 166
 167         description = clip.get('caption') or clip.get('description')
 168         thumbnail = clip.get('thumbnail')
 169         duration = int_or_none(clip.get('duration'))
 170         timestamp = unified_timestamp(clip.get('originalPublishDate'))
 171
 172         return {
 173             'id': video_id,
 174             'title': title,
 175             'description': description,
 176             'thumbnail': thumbnail,
 177             'timestamp': timestamp,
 178             'duration': duration,
 179             'formats': formats,
 180         }
 181
 182
 183 class ESPNArticleIE(InfoExtractor):
 184     _VALID_URL = r'https?://(?:espn\.go|(?:www\.)?espn)\.com/(?:[^/]+/)*(?P<id>[^/]+)'
 185     _TESTS = [{
 186         'url': 'http://espn.go.com/nba/recap?gameId=400793786',
 187         'only_matching': True,
 188     }, {
 189         'url': 'http://espn.go.com/blog/golden-state-warriors/post/_/id/593/how-warriors-rapidly-regained-a-winning-edge',
 190         'only_matching': True,
 191     }, {
 192         'url': 'http://espn.go.com/sports/endurance/story/_/id/12893522/dzhokhar-tsarnaev-sentenced-role-boston-marathon-bombings',
 193         'only_matching': True,
 194     }, {
 195         'url': 'http://espn.go.com/nba/playoffs/2015/story/_/id/12887571/john-wall-washington-wizards-no-swelling-left-hand-wrist-game-5-return',
 196         'only_matching': True,
 197     }]
 198
 199     @classmethod
 200     def suitable(cls, url):
 201         return False if ESPNIE.suitable(url) else super(ESPNArticleIE, cls).suitable(url)
 202
 203     def _real_extract(self, url):
 204         video_id = self._match_id(url)
 205
 206         webpage = self._download_webpage(url, video_id)
 207
 208         video_id = self._search_regex(
 209             r'class=(["\']).*?video-play-button.*?\1[^>]+data-id=["\'](?P<id>\d+)',
 210             webpage, 'video id', group='id')
 211
 212         return self.url_result(
 213             'http://espn.go.com/video/clip?id=%s' % video_id, ESPNIE.ie_key())
 214
 215
 216 class FiveThirtyEightIE(InfoExtractor):
 217     _VALID_URL = r'https?://(?:www\.)?fivethirtyeight\.com/features/(?P<id>[^/?#]+)'
 218     _TEST = {
 219         'url': 'http://fivethirtyeight.com/features/how-the-6-8-raiders-can-still-make-the-playoffs/',
 220         'info_dict': {
 221             'id': '56032156',
 222             'ext': 'flv',
 223             'title': 'FiveThirtyEight: The Raiders can still make the playoffs',
 224             'description': 'Neil Paine breaks down the simplest scenario that will put the Raiders into the playoffs at 8-8.',
 225         },
 226         'params': {
 227             'skip_download': True,
 228         },
 229     }
 230
 231     def _real_extract(self, url):
 232         video_id = self._match_id(url)
 233
 234         webpage = self._download_webpage(url, video_id)
 235
 236         embed_url = self._search_regex(
 237             r'<iframe[^>]+src=["\'](https?://fivethirtyeight\.abcnews\.go\.com/video/embed/\d+/\d+)',
 238             webpage, 'embed url')
 239
 240         return self.url_result(embed_url, 'AbcNewsVideo')
 241
 242
 243 class ESPNCricInfoIE(InfoExtractor):
 244     _VALID_URL = r'https?://(?:www\.)?espncricinfo\.com/video/[^#$&?/]+-(?P<id>\d+)'
 245     _TESTS = [{
 246         'url': 'https://www.espncricinfo.com/video/finch-chasing-comes-with-risks-despite-world-cup-trend-1289135',
 247         'info_dict': {
 248             'id': '1289135',
 249             'ext': 'mp4',
 250             'title': 'Finch: Chasing comes with \'risks\' despite World Cup trend',
 251             'description': 'md5:ea32373303e25efbb146efdfc8a37829',
 252             'upload_date': '20211113',
 253             'duration': 96,
 254         },
 255         'params': {'skip_download': True}
 256     }]
 257
 258     def _real_extract(self, url):
 259         id = self._match_id(url)
 260         data_json = self._download_json(f'https://hs-consumer-api.espncricinfo.com/v1/pages/video/video-details?videoId={id}', id)['video']
 261         formats, subtitles = [], {}
 262         for item in data_json.get('playbacks') or []:
 263             if item.get('type') == 'HLS' and item.get('url'):
 264                 m3u8_frmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(item['url'], id)
 265                 formats.extend(m3u8_frmts)
 266                 subtitles = self._merge_subtitles(subtitles, m3u8_subs)
 267             elif item.get('type') == 'AUDIO' and item.get('url'):
 268                 formats.append({
 269                     'url': item['url'],
 270                     'vcodec': 'none',
 271                 })
 272         self._sort_formats(formats)
 273         return {
 274             'id': id,
 275             'title': data_json.get('title'),
 276             'description': data_json.get('summary'),
 277             'upload_date': unified_strdate(dict_get(data_json, ('publishedAt', 'recordedAt'))),
 278             'duration': data_json.get('duration'),
 279             'formats': formats,
 280             'subtitles': subtitles,
 281         }