yt_dlp/extractor/nfl.py

   1 import base64
   2 import json
   3 import re
   4 import time
   5 import uuid
   6
   7 from .anvato import AnvatoIE
   8 from .common import InfoExtractor
   9 from ..utils import (
  10     ExtractorError,
  11     clean_html,
  12     determine_ext,
  13     get_element_by_class,
  14     traverse_obj,
  15     urlencode_postdata,
  16 )
  17
  18
  19 class NFLBaseIE(InfoExtractor):
  20     _VALID_URL_BASE = r'''(?x)
  21                     https?://
  22                         (?P<host>
  23                             (?:www\.)?
  24                             (?:
  25                                 (?:
  26                                     nfl|
  27                                     buffalobills|
  28                                     miamidolphins|
  29                                     patriots|
  30                                     newyorkjets|
  31                                     baltimoreravens|
  32                                     bengals|
  33                                     clevelandbrowns|
  34                                     steelers|
  35                                     houstontexans|
  36                                     colts|
  37                                     jaguars|
  38                                     (?:titansonline|tennesseetitans)|
  39                                     denverbroncos|
  40                                     (?:kc)?chiefs|
  41                                     raiders|
  42                                     chargers|
  43                                     dallascowboys|
  44                                     giants|
  45                                     philadelphiaeagles|
  46                                     (?:redskins|washingtonfootball)|
  47                                     chicagobears|
  48                                     detroitlions|
  49                                     packers|
  50                                     vikings|
  51                                     atlantafalcons|
  52                                     panthers|
  53                                     neworleanssaints|
  54                                     buccaneers|
  55                                     azcardinals|
  56                                     (?:stlouis|the)rams|
  57                                     49ers|
  58                                     seahawks
  59                                 )\.com|
  60                                 .+?\.clubs\.nfl\.com
  61                             )
  62                         )/
  63                     '''
  64     _VIDEO_CONFIG_REGEX = r'<script[^>]+id="[^"]*video-config-[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}[^"]*"[^>]*>\s*({.+});?\s*</script>'
  65     _ANVATO_PREFIX = 'anvato:GXvEgwyJeWem8KCYXfeoHWknwP48Mboj:'
  66
  67     def _parse_video_config(self, video_config, display_id):
  68         video_config = self._parse_json(video_config, display_id)
  69         item = video_config['playlist'][0]
  70         mcp_id = item.get('mcpID')
  71         if mcp_id:
  72             info = self.url_result(f'{self._ANVATO_PREFIX}{mcp_id}', AnvatoIE, mcp_id)
  73         else:
  74             media_id = item.get('id') or item['entityId']
  75             title = item.get('title')
  76             item_url = item['url']
  77             info = {'id': media_id}
  78             ext = determine_ext(item_url)
  79             if ext == 'm3u8':
  80                 info['formats'] = self._extract_m3u8_formats(item_url, media_id, 'mp4')
  81             else:
  82                 info['url'] = item_url
  83                 if item.get('audio') is True:
  84                     info['vcodec'] = 'none'
  85             is_live = video_config.get('live') is True
  86             thumbnails = None
  87             image_url = item.get(item.get('imageSrc')) or item.get(item.get('posterImage'))
  88             if image_url:
  89                 thumbnails = [{
  90                     'url': image_url,
  91                     'ext': determine_ext(image_url, 'jpg'),
  92                 }]
  93             info.update({
  94                 'title': title,
  95                 'is_live': is_live,
  96                 'description': clean_html(item.get('description')),
  97                 'thumbnails': thumbnails,
  98             })
  99         return info
 100
 101
 102 class NFLIE(NFLBaseIE):
 103     IE_NAME = 'nfl.com'
 104     _VALID_URL = NFLBaseIE._VALID_URL_BASE + r'(?:videos?|listen|audio)/(?P<id>[^/#?&]+)'
 105     _TESTS = [{
 106         'url': 'https://www.nfl.com/videos/baker-mayfield-s-game-changing-plays-from-3-td-game-week-14',
 107         'info_dict': {
 108             'id': '899441',
 109             'ext': 'mp4',
 110             'title': "Baker Mayfield's game-changing plays from 3-TD game Week 14",
 111             'description': 'md5:85e05a3cc163f8c344340f220521136d',
 112             'upload_date': '20201215',
 113             'timestamp': 1608009755,
 114             'thumbnail': r're:^https?://.*\.jpg$',
 115             'uploader': 'NFL',
 116             'tags': 'count:6',
 117             'duration': 157,
 118             'categories': 'count:3',
 119         }
 120     }, {
 121         'url': 'https://www.chiefs.com/listen/patrick-mahomes-travis-kelce-react-to-win-over-dolphins-the-breakdown',
 122         'md5': '6886b32c24b463038c760ceb55a34566',
 123         'info_dict': {
 124             'id': 'd87e8790-3e14-11eb-8ceb-ff05c2867f99',
 125             'ext': 'mp3',
 126             'title': 'Patrick Mahomes, Travis Kelce React to Win Over Dolphins | The Breakdown',
 127             'description': 'md5:12ada8ee70e6762658c30e223e095075',
 128         },
 129         'skip': 'HTTP Error 404: Not Found',
 130     }, {
 131         'url': 'https://www.buffalobills.com/video/buffalo-bills-military-recognition-week-14',
 132         'only_matching': True,
 133     }, {
 134         'url': 'https://www.raiders.com/audio/instant-reactions-raiders-week-14-loss-to-indianapolis-colts-espn-jason-fitz',
 135         'only_matching': True,
 136     }]
 137
 138     def _real_extract(self, url):
 139         display_id = self._match_id(url)
 140         webpage = self._download_webpage(url, display_id)
 141         return self._parse_video_config(self._search_regex(
 142             self._VIDEO_CONFIG_REGEX, webpage, 'video config'), display_id)
 143
 144
 145 class NFLArticleIE(NFLBaseIE):
 146     IE_NAME = 'nfl.com:article'
 147     _VALID_URL = NFLBaseIE._VALID_URL_BASE + r'news/(?P<id>[^/#?&]+)'
 148     _TEST = {
 149         'url': 'https://www.buffalobills.com/news/the-only-thing-we-ve-earned-is-the-noise-bills-coaches-discuss-handling-rising-e',
 150         'info_dict': {
 151             'id': 'the-only-thing-we-ve-earned-is-the-noise-bills-coaches-discuss-handling-rising-e',
 152             'title': "'The only thing we've earned is the noise' | Bills coaches discuss handling rising expectations",
 153         },
 154         'playlist_count': 4,
 155     }
 156
 157     def _real_extract(self, url):
 158         display_id = self._match_id(url)
 159         webpage = self._download_webpage(url, display_id)
 160         entries = []
 161         for video_config in re.findall(self._VIDEO_CONFIG_REGEX, webpage):
 162             entries.append(self._parse_video_config(video_config, display_id))
 163         title = clean_html(get_element_by_class(
 164             'nfl-c-article__title', webpage)) or self._html_search_meta(
 165             ['og:title', 'twitter:title'], webpage)
 166         return self.playlist_result(entries, display_id, title)
 167
 168
 169 class NFLPlusReplayIE(NFLBaseIE):
 170     IE_NAME = 'nfl.com:plus:replay'
 171     _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/games/[\w-]+/(?P<id>\d+)'
 172     _TESTS = [{
 173         'url': 'https://www.nfl.com/plus/games/giants-at-vikings-2022-post-1/1572108',
 174         'info_dict': {
 175             'id': '1572108',
 176             'ext': 'mp4',
 177             'title': 'New York Giants at Minnesota Vikings',
 178             'description': 'New York Giants play the Minnesota Vikings at U.S. Bank Stadium on January 15, 2023',
 179             'uploader': 'NFL',
 180             'upload_date': '20230116',
 181             'timestamp': 1673864520,
 182             'duration': 7157,
 183             'categories': ['Game Highlights'],
 184             'tags': ['Minnesota Vikings', 'New York Giants', 'Minnesota Vikings vs. New York Giants'],
 185             'thumbnail': r're:^https?://.*\.jpg',
 186         },
 187         'params': {'skip_download': 'm3u8'},
 188     }]
 189
 190     def _real_extract(self, url):
 191         video_id = self._match_id(url)
 192         return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id)
 193
 194
 195 class NFLPlusEpisodeIE(NFLBaseIE):
 196     IE_NAME = 'nfl.com:plus:episode'
 197     _VALID_URL = r'https?://(?:www\.)?nfl.com/plus/episodes/(?P<id>[\w-]+)'
 198     _TESTS = [{
 199         'note': 'premium content',
 200         'url': 'https://www.nfl.com/plus/episodes/kurt-s-qb-insider-conference-championships',
 201         'info_dict': {
 202             'id': '1576832',
 203             'ext': 'mp4',
 204             'title': 'Kurt\'s QB Insider: Conference Championships',
 205             'description': 'md5:944f7fab56f7a37430bf8473f5473857',
 206             'uploader': 'NFL',
 207             'upload_date': '20230127',
 208             'timestamp': 1674782760,
 209             'duration': 730,
 210             'categories': ['Analysis'],
 211             'tags': ['Cincinnati Bengals at Kansas City Chiefs (2022-POST-3)'],
 212             'thumbnail': r're:^https?://.*\.jpg',
 213         },
 214         'params': {'skip_download': 'm3u8'},
 215     }]
 216
 217     _CLIENT_DATA = {
 218         'clientKey': '4cFUW6DmwJpzT9L7LrG3qRAcABG5s04g',
 219         'clientSecret': 'CZuvCL49d9OwfGsR',
 220         'deviceId': str(uuid.uuid4()),
 221         'deviceInfo': base64.b64encode(json.dumps({
 222             'model': 'desktop',
 223             'version': 'Chrome',
 224             'osName': 'Windows',
 225             'osVersion': '10.0',
 226         }, separators=(',', ':')).encode()).decode(),
 227         'networkType': 'other',
 228         'nflClaimGroupsToAdd': [],
 229         'nflClaimGroupsToRemove': [],
 230     }
 231     _ACCOUNT_INFO = {}
 232     _API_KEY = None
 233
 234     _TOKEN = None
 235     _TOKEN_EXPIRY = 0
 236
 237     def _get_account_info(self, url, video_id):
 238         cookies = self._get_cookies('https://www.nfl.com/')
 239         login_token = traverse_obj(cookies, (
 240             (f'glt_{self._API_KEY}', f'gig_loginToken_{self._API_KEY}',
 241              lambda k, _: k.startswith('glt_') or k.startswith('gig_loginToken_')),
 242             {lambda x: x.value}), get_all=False)
 243         if not login_token:
 244             self.raise_login_required()
 245
 246         account = self._download_json(
 247             'https://auth-id.nfl.com/accounts.getAccountInfo', video_id,
 248             note='Downloading account info', data=urlencode_postdata({
 249                 'include': 'profile,data',
 250                 'lang': 'en',
 251                 'APIKey': self._API_KEY,
 252                 'sdk': 'js_latest',
 253                 'login_token': login_token,
 254                 'authMode': 'cookie',
 255                 'pageURL': url,
 256                 'sdkBuild': traverse_obj(cookies, (
 257                     'gig_canary_ver', {lambda x: x.value.partition('-')[0]}), default='13642'),
 258                 'format': 'json',
 259             }), headers={'Content-Type': 'application/x-www-form-urlencoded'})
 260
 261         self._ACCOUNT_INFO = traverse_obj(account, {
 262             'signatureTimestamp': 'signatureTimestamp',
 263             'uid': 'UID',
 264             'uidSignature': 'UIDSignature',
 265         })
 266
 267         if len(self._ACCOUNT_INFO) != 3:
 268             raise ExtractorError('Failed to retrieve account info with provided cookies', expected=True)
 269
 270     def _get_auth_token(self, url, video_id):
 271         if not self._ACCOUNT_INFO:
 272             self._get_account_info(url, video_id)
 273
 274         token = self._download_json(
 275             'https://api.nfl.com/identity/v3/token%s' % (
 276                 '/refresh' if self._ACCOUNT_INFO.get('refreshToken') else ''),
 277             video_id, headers={'Content-Type': 'application/json'}, note='Downloading access token',
 278             data=json.dumps({**self._CLIENT_DATA, **self._ACCOUNT_INFO}, separators=(',', ':')).encode())
 279
 280         self._TOKEN = token['accessToken']
 281         self._TOKEN_EXPIRY = token['expiresIn']
 282         self._ACCOUNT_INFO['refreshToken'] = token['refreshToken']
 283
 284     def _real_extract(self, url):
 285         slug = self._match_id(url)
 286
 287         if not self._API_KEY:
 288             webpage = self._download_webpage(url, slug, fatal=False) or ''
 289             self._API_KEY = self._search_regex(
 290                 r'window\.gigyaApiKey=["\'](\w+)["\'];', webpage, 'API key',
 291                 default='3_Qa8TkWpIB8ESCBT8tY2TukbVKgO5F6BJVc7N1oComdwFzI7H2L9NOWdm11i_BY9f')
 292
 293         if not self._TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
 294             self._get_auth_token(url, slug)
 295
 296         video_id = self._download_json(
 297             f'https://api.nfl.com/content/v1/videos/episodes/{slug}', slug, headers={
 298                 'Authorization': f'Bearer {self._TOKEN}',
 299             })['mcpPlaybackId']
 300
 301         return self.url_result(f'{self._ANVATO_PREFIX}{video_id}', AnvatoIE, video_id)