yt_dlp/extractor/nbc.py

   1 import base64
   2 import json
   3 import re
   4
   5 from .common import InfoExtractor
   6 from .theplatform import ThePlatformIE, default_ns
   7 from .adobepass import AdobePassIE
   8 from ..compat import compat_urllib_parse_unquote
   9 from ..utils import (
  10     ExtractorError,
  11     HEADRequest,
  12     RegexNotFoundError,
  13     UserNotLive,
  14     clean_html,
  15     int_or_none,
  16     parse_age_limit,
  17     parse_duration,
  18     smuggle_url,
  19     traverse_obj,
  20     try_get,
  21     unescapeHTML,
  22     unified_timestamp,
  23     update_url_query,
  24     url_basename,
  25     xpath_attr,
  26 )
  27
  28
  29 class NBCIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
  30     _VALID_URL = r'https?(?P<permalink>://(?:www\.)?nbc\.com/(?:classic-tv/)?[^/]+/video/[^/]+/(?P<id>(?:NBCE|n)?\d+))'
  31
  32     _TESTS = [
  33         {
  34             'url': 'http://www.nbc.com/the-tonight-show/video/jimmy-fallon-surprises-fans-at-ben-jerrys/2848237',
  35             'info_dict': {
  36                 'id': '2848237',
  37                 'ext': 'mp4',
  38                 'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
  39                 'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
  40                 'timestamp': 1424246400,
  41                 'upload_date': '20150218',
  42                 'uploader': 'NBCU-COM',
  43                 'episode': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
  44                 'episode_number': 86,
  45                 'season': 'Season 2',
  46                 'season_number': 2,
  47                 'series': 'Tonight Show: Jimmy Fallon',
  48                 'duration': 237.0,
  49                 'chapters': 'count:1',
  50                 'tags': 'count:4',
  51                 'thumbnail': r're:https?://.+\.jpg',
  52             },
  53             'params': {
  54                 'skip_download': 'm3u8',
  55             },
  56         },
  57         {
  58             'url': 'http://www.nbc.com/saturday-night-live/video/star-wars-teaser/2832821',
  59             'info_dict': {
  60                 'id': '2832821',
  61                 'ext': 'mp4',
  62                 'title': 'Star Wars Teaser',
  63                 'description': 'md5:0b40f9cbde5b671a7ff62fceccc4f442',
  64                 'timestamp': 1417852800,
  65                 'upload_date': '20141206',
  66                 'uploader': 'NBCU-COM',
  67             },
  68             'skip': 'page not found',
  69         },
  70         {
  71             # HLS streams requires the 'hdnea3' cookie
  72             'url': 'http://www.nbc.com/Kings/video/goliath/n1806',
  73             'info_dict': {
  74                 'id': '101528f5a9e8127b107e98c5e6ce4638',
  75                 'ext': 'mp4',
  76                 'title': 'Goliath',
  77                 'description': 'When an unknown soldier saves the life of the King\'s son in battle, he\'s thrust into the limelight and politics of the kingdom.',
  78                 'timestamp': 1237100400,
  79                 'upload_date': '20090315',
  80                 'uploader': 'NBCU-COM',
  81             },
  82             'skip': 'page not found',
  83         },
  84         {
  85             # manifest url does not have extension
  86             'url': 'https://www.nbc.com/the-golden-globe-awards/video/oprah-winfrey-receives-cecil-b-de-mille-award-at-the-2018-golden-globes/3646439',
  87             'info_dict': {
  88                 'id': '3646439',
  89                 'ext': 'mp4',
  90                 'title': 'Oprah Winfrey Receives Cecil B. de Mille Award at the 2018 Golden Globes',
  91                 'episode': 'Oprah Winfrey Receives Cecil B. de Mille Award at the 2018 Golden Globes',
  92                 'episode_number': 1,
  93                 'season': 'Season 75',
  94                 'season_number': 75,
  95                 'series': 'The Golden Globe Awards',
  96                 'description': 'Oprah Winfrey receives the Cecil B. de Mille Award at the 75th Annual Golden Globe Awards.',
  97                 'uploader': 'NBCU-COM',
  98                 'upload_date': '20180107',
  99                 'timestamp': 1515312000,
 100                 'duration': 570.0,
 101                 'tags': 'count:8',
 102                 'thumbnail': r're:https?://.+\.jpg',
 103                 'chapters': 'count:1',
 104             },
 105             'params': {
 106                 'skip_download': 'm3u8',
 107             },
 108         },
 109         {
 110             # new video_id format
 111             'url': 'https://www.nbc.com/quantum-leap/video/bens-first-leap-nbcs-quantum-leap/NBCE125189978',
 112             'info_dict': {
 113                 'id': 'NBCE125189978',
 114                 'ext': 'mp4',
 115                 'title': 'Ben\'s First Leap | NBC\'s Quantum Leap',
 116                 'description': 'md5:a82762449b7ec4bb83291a7b355ebf8e',
 117                 'uploader': 'NBCU-COM',
 118                 'series': 'Quantum Leap',
 119                 'season': 'Season 1',
 120                 'season_number': 1,
 121                 'episode': 'Ben\'s First Leap | NBC\'s Quantum Leap',
 122                 'episode_number': 1,
 123                 'duration': 170.171,
 124                 'chapters': [],
 125                 'timestamp': 1663956155,
 126                 'upload_date': '20220923',
 127                 'tags': 'count:10',
 128                 'age_limit': 0,
 129                 'thumbnail': r're:https?://.+\.jpg',
 130             },
 131             'expected_warnings': ['Ignoring subtitle tracks'],
 132             'params': {
 133                 'skip_download': 'm3u8',
 134             },
 135         },
 136         {
 137             'url': 'https://www.nbc.com/classic-tv/charles-in-charge/video/charles-in-charge-pilot/n3310',
 138             'only_matching': True,
 139         },
 140         {
 141             # Percent escaped url
 142             'url': 'https://www.nbc.com/up-all-night/video/day-after-valentine%27s-day/n2189',
 143             'only_matching': True,
 144         }
 145     ]
 146
 147     def _real_extract(self, url):
 148         permalink, video_id = self._match_valid_url(url).groups()
 149         permalink = 'http' + compat_urllib_parse_unquote(permalink)
 150         video_data = self._download_json(
 151             'https://friendship.nbc.co/v2/graphql', video_id, query={
 152                 'query': '''query bonanzaPage(
 153   $app: NBCUBrands! = nbc
 154   $name: String!
 155   $oneApp: Boolean
 156   $platform: SupportedPlatforms! = web
 157   $type: EntityPageType! = VIDEO
 158   $userId: String!
 159 ) {
 160   bonanzaPage(
 161     app: $app
 162     name: $name
 163     oneApp: $oneApp
 164     platform: $platform
 165     type: $type
 166     userId: $userId
 167   ) {
 168     metadata {
 169       ... on VideoPageData {
 170         description
 171         episodeNumber
 172         keywords
 173         locked
 174         mpxAccountId
 175         mpxGuid
 176         rating
 177         resourceId
 178         seasonNumber
 179         secondaryTitle
 180         seriesShortTitle
 181       }
 182     }
 183   }
 184 }''',
 185                 'variables': json.dumps({
 186                     'name': permalink,
 187                     'oneApp': True,
 188                     'userId': '0',
 189                 }),
 190             })['data']['bonanzaPage']['metadata']
 191         query = {
 192             'mbr': 'true',
 193             'manifest': 'm3u',
 194             'switch': 'HLSServiceSecure',
 195         }
 196         video_id = video_data['mpxGuid']
 197         tp_path = 'NnzsPC/media/guid/%s/%s' % (video_data.get('mpxAccountId') or '2410887629', video_id)
 198         tpm = self._download_theplatform_metadata(tp_path, video_id)
 199         title = tpm.get('title') or video_data.get('secondaryTitle')
 200         if video_data.get('locked'):
 201             resource = self._get_mvpd_resource(
 202                 video_data.get('resourceId') or 'nbcentertainment',
 203                 title, video_id, video_data.get('rating'))
 204             query['auth'] = self._extract_mvpd_auth(
 205                 url, video_id, 'nbcentertainment', resource)
 206         theplatform_url = smuggle_url(update_url_query(
 207             'http://link.theplatform.com/s/NnzsPC/media/guid/%s/%s' % (video_data.get('mpxAccountId') or '2410887629', video_id),
 208             query), {'force_smil_url': True})
 209
 210         # Empty string or 0 can be valid values for these. So the check must be `is None`
 211         description = video_data.get('description')
 212         if description is None:
 213             description = tpm.get('description')
 214         episode_number = int_or_none(video_data.get('episodeNumber'))
 215         if episode_number is None:
 216             episode_number = int_or_none(tpm.get('nbcu$airOrder'))
 217         rating = video_data.get('rating')
 218         if rating is None:
 219             try_get(tpm, lambda x: x['ratings'][0]['rating'])
 220         season_number = int_or_none(video_data.get('seasonNumber'))
 221         if season_number is None:
 222             season_number = int_or_none(tpm.get('nbcu$seasonNumber'))
 223         series = video_data.get('seriesShortTitle')
 224         if series is None:
 225             series = tpm.get('nbcu$seriesShortTitle')
 226         tags = video_data.get('keywords')
 227         if tags is None or len(tags) == 0:
 228             tags = tpm.get('keywords')
 229
 230         return {
 231             '_type': 'url_transparent',
 232             'age_limit': parse_age_limit(rating),
 233             'description': description,
 234             'episode': title,
 235             'episode_number': episode_number,
 236             'id': video_id,
 237             'ie_key': 'ThePlatform',
 238             'season_number': season_number,
 239             'series': series,
 240             'tags': tags,
 241             'title': title,
 242             'url': theplatform_url,
 243         }
 244
 245
 246 class NBCSportsVPlayerIE(InfoExtractor):
 247     _VALID_URL_BASE = r'https?://(?:vplayer\.nbcsports\.com|(?:www\.)?nbcsports\.com/vplayer)/'
 248     _VALID_URL = _VALID_URL_BASE + r'(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 249     _EMBED_REGEX = [r'(?:iframe[^>]+|var video|div[^>]+data-(?:mpx-)?)[sS]rc\s?=\s?"(?P<url>%s[^\"]+)' % _VALID_URL_BASE]
 250
 251     _TESTS = [{
 252         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/9CsDKds0kvHI',
 253         'info_dict': {
 254             'id': '9CsDKds0kvHI',
 255             'ext': 'mp4',
 256             'description': 'md5:df390f70a9ba7c95ff1daace988f0d8d',
 257             'title': 'Tyler Kalinoski hits buzzer-beater to lift Davidson',
 258             'timestamp': 1426270238,
 259             'upload_date': '20150313',
 260             'uploader': 'NBCU-SPORTS',
 261             'duration': 72.818,
 262             'chapters': [],
 263             'thumbnail': r're:^https?://.*\.jpg$'
 264         }
 265     }, {
 266         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/media/PEgOtlNcC_y2',
 267         'only_matching': True,
 268     }, {
 269         'url': 'https://www.nbcsports.com/vplayer/p/BxmELC/nbcsports/select/PHJSaFWbrTY9?form=html&autoPlay=true',
 270         'only_matching': True,
 271     }]
 272
 273     def _real_extract(self, url):
 274         video_id = self._match_id(url)
 275         webpage = self._download_webpage(url, video_id)
 276         theplatform_url = self._html_search_regex(r'tp:releaseUrl="(.+?)"', webpage, 'url')
 277         return self.url_result(theplatform_url, 'ThePlatform')
 278
 279
 280 class NBCSportsIE(InfoExtractor):
 281     _VALID_URL = r'https?://(?:www\.)?nbcsports\.com//?(?!vplayer/)(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
 282
 283     _TESTS = [{
 284         # iframe src
 285         'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
 286         'info_dict': {
 287             'id': 'PHJSaFWbrTY9',
 288             'ext': 'mp4',
 289             'title': 'Tom Izzo, Michigan St. has \'so much respect\' for Duke',
 290             'description': 'md5:ecb459c9d59e0766ac9c7d5d0eda8113',
 291             'uploader': 'NBCU-SPORTS',
 292             'upload_date': '20150330',
 293             'timestamp': 1427726529,
 294             'chapters': [],
 295             'thumbnail': 'https://hdliveextra-a.akamaihd.net/HD/image_sports/NBCU_Sports_Group_-_nbcsports/253/303/izzodps.jpg',
 296             'duration': 528.395,
 297         }
 298     }, {
 299         # data-mpx-src
 300         'url': 'https://www.nbcsports.com/philadelphia/philadelphia-phillies/bruce-bochy-hector-neris-hes-idiot',
 301         'only_matching': True,
 302     }, {
 303         # data-src
 304         'url': 'https://www.nbcsports.com/boston/video/report-card-pats-secondary-no-match-josh-allen',
 305         'only_matching': True,
 306     }]
 307
 308     def _real_extract(self, url):
 309         video_id = self._match_id(url)
 310         webpage = self._download_webpage(url, video_id)
 311         return self.url_result(
 312             NBCSportsVPlayerIE._extract_url(webpage), 'NBCSportsVPlayer')
 313
 314
 315 class NBCSportsStreamIE(AdobePassIE):
 316     _VALID_URL = r'https?://stream\.nbcsports\.com/.+?\bpid=(?P<id>\d+)'
 317     _TEST = {
 318         'url': 'http://stream.nbcsports.com/nbcsn/generic?pid=206559',
 319         'info_dict': {
 320             'id': '206559',
 321             'ext': 'mp4',
 322             'title': 'Amgen Tour of California Women\'s Recap',
 323             'description': 'md5:66520066b3b5281ada7698d0ea2aa894',
 324         },
 325         'params': {
 326             # m3u8 download
 327             'skip_download': True,
 328         },
 329         'skip': 'Requires Adobe Pass Authentication',
 330     }
 331
 332     def _real_extract(self, url):
 333         video_id = self._match_id(url)
 334         live_source = self._download_json(
 335             'http://stream.nbcsports.com/data/live_sources_%s.json' % video_id,
 336             video_id)
 337         video_source = live_source['videoSources'][0]
 338         title = video_source['title']
 339         source_url = None
 340         for k in ('source', 'msl4source', 'iossource', 'hlsv4'):
 341             sk = k + 'Url'
 342             source_url = video_source.get(sk) or video_source.get(sk + 'Alt')
 343             if source_url:
 344                 break
 345         else:
 346             source_url = video_source['ottStreamUrl']
 347         is_live = video_source.get('type') == 'live' or video_source.get('status') == 'Live'
 348         resource = self._get_mvpd_resource('nbcsports', title, video_id, '')
 349         token = self._extract_mvpd_auth(url, video_id, 'nbcsports', resource)
 350         tokenized_url = self._download_json(
 351             'https://token.playmakerservices.com/cdn',
 352             video_id, data=json.dumps({
 353                 'requestorId': 'nbcsports',
 354                 'pid': video_id,
 355                 'application': 'NBCSports',
 356                 'version': 'v1',
 357                 'platform': 'desktop',
 358                 'cdn': 'akamai',
 359                 'url': video_source['sourceUrl'],
 360                 'token': base64.b64encode(token.encode()).decode(),
 361                 'resourceId': base64.b64encode(resource.encode()).decode(),
 362             }).encode())['tokenizedUrl']
 363         formats = self._extract_m3u8_formats(tokenized_url, video_id, 'mp4')
 364         return {
 365             'id': video_id,
 366             'title': title,
 367             'description': live_source.get('description'),
 368             'formats': formats,
 369             'is_live': is_live,
 370         }
 371
 372
 373 class NBCNewsIE(ThePlatformIE):  # XXX: Do not subclass from concrete IE
 374     _VALID_URL = r'(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/([^/]+/)*(?:.*-)?(?P<id>[^/?]+)'
 375     _EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1']
 376
 377     _TESTS = [
 378         {
 379             'url': 'http://www.nbcnews.com/watch/nbcnews-com/how-twitter-reacted-to-the-snowden-interview-269389891880',
 380             'md5': 'cf4bc9e6ce0130f00f545d80ecedd4bf',
 381             'info_dict': {
 382                 'id': '269389891880',
 383                 'ext': 'mp4',
 384                 'title': 'How Twitter Reacted To The Snowden Interview',
 385                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
 386                 'timestamp': 1401363060,
 387                 'upload_date': '20140529',
 388             },
 389         },
 390         {
 391             'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
 392             'md5': 'fdbf39ab73a72df5896b6234ff98518a',
 393             'info_dict': {
 394                 'id': '529953347624',
 395                 'ext': 'mp4',
 396                 'title': 'FULL EPISODE: Family Business',
 397                 'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
 398             },
 399             'skip': 'This page is unavailable.',
 400         },
 401         {
 402             'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
 403             'md5': '8eb831eca25bfa7d25ddd83e85946548',
 404             'info_dict': {
 405                 'id': '394064451844',
 406                 'ext': 'mp4',
 407                 'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
 408                 'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
 409                 'timestamp': 1423104900,
 410                 'upload_date': '20150205',
 411             },
 412         },
 413         {
 414             'url': 'http://www.nbcnews.com/business/autos/volkswagen-11-million-vehicles-could-have-suspect-software-emissions-scandal-n431456',
 415             'md5': '4a8c4cec9e1ded51060bdda36ff0a5c0',
 416             'info_dict': {
 417                 'id': 'n431456',
 418                 'ext': 'mp4',
 419                 'title': "Volkswagen U.S. Chief:  We 'Totally Screwed Up'",
 420                 'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
 421                 'upload_date': '20150922',
 422                 'timestamp': 1442917800,
 423             },
 424         },
 425         {
 426             'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
 427             'md5': '118d7ca3f0bea6534f119c68ef539f71',
 428             'info_dict': {
 429                 'id': '669831235788',
 430                 'ext': 'mp4',
 431                 'title': 'See the aurora borealis from space in stunning new NASA video',
 432                 'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
 433                 'upload_date': '20160420',
 434                 'timestamp': 1461152093,
 435             },
 436         },
 437         {
 438             'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
 439             'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
 440             'info_dict': {
 441                 'id': '314487875924',
 442                 'ext': 'mp4',
 443                 'title': 'The chaotic GOP immigration vote',
 444                 'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
 445                 'thumbnail': r're:^https?://.*\.jpg$',
 446                 'timestamp': 1406937606,
 447                 'upload_date': '20140802',
 448             },
 449         },
 450         {
 451             'url': 'http://www.nbcnews.com/watch/dateline/full-episode--deadly-betrayal-386250819952',
 452             'only_matching': True,
 453         },
 454         {
 455             # From http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html
 456             'url': 'http://www.nbcnews.com/widget/video-embed/701714499682',
 457             'only_matching': True,
 458         },
 459     ]
 460
 461     def _real_extract(self, url):
 462         video_id = self._match_id(url)
 463         webpage = self._download_webpage(url, video_id)
 464
 465         data = self._search_nextjs_data(webpage, video_id)['props']['initialState']
 466         video_data = try_get(data, lambda x: x['video']['current'], dict)
 467         if not video_data:
 468             video_data = data['article']['content'][0]['primaryMedia']['video']
 469         title = video_data['headline']['primary']
 470
 471         formats = []
 472         for va in video_data.get('videoAssets', []):
 473             public_url = va.get('publicUrl')
 474             if not public_url:
 475                 continue
 476             if '://link.theplatform.com/' in public_url:
 477                 public_url = update_url_query(public_url, {'format': 'redirect'})
 478             format_id = va.get('format')
 479             if format_id == 'M3U':
 480                 formats.extend(self._extract_m3u8_formats(
 481                     public_url, video_id, 'mp4', 'm3u8_native',
 482                     m3u8_id=format_id, fatal=False))
 483                 continue
 484             tbr = int_or_none(va.get('bitrate'), 1000)
 485             if tbr:
 486                 format_id += '-%d' % tbr
 487             formats.append({
 488                 'format_id': format_id,
 489                 'url': public_url,
 490                 'width': int_or_none(va.get('width')),
 491                 'height': int_or_none(va.get('height')),
 492                 'tbr': tbr,
 493                 'ext': 'mp4',
 494             })
 495
 496         subtitles = {}
 497         closed_captioning = video_data.get('closedCaptioning')
 498         if closed_captioning:
 499             for cc_url in closed_captioning.values():
 500                 if not cc_url:
 501                     continue
 502                 subtitles.setdefault('en', []).append({
 503                     'url': cc_url,
 504                 })
 505
 506         return {
 507             'id': video_id,
 508             'title': title,
 509             'description': try_get(video_data, lambda x: x['description']['primary']),
 510             'thumbnail': try_get(video_data, lambda x: x['primaryImage']['url']['primary']),
 511             'duration': parse_duration(video_data.get('duration')),
 512             'timestamp': unified_timestamp(video_data.get('datePublished')),
 513             'formats': formats,
 514             'subtitles': subtitles,
 515         }
 516
 517
 518 class NBCOlympicsIE(InfoExtractor):
 519     IE_NAME = 'nbcolympics'
 520     _VALID_URL = r'https?://www\.nbcolympics\.com/videos?/(?P<id>[0-9a-z-]+)'
 521
 522     _TEST = {
 523         # Geo-restricted to US
 524         'url': 'http://www.nbcolympics.com/video/justin-roses-son-leo-was-tears-after-his-dad-won-gold',
 525         'md5': '54fecf846d05429fbaa18af557ee523a',
 526         'info_dict': {
 527             'id': 'WjTBzDXx5AUq',
 528             'display_id': 'justin-roses-son-leo-was-tears-after-his-dad-won-gold',
 529             'ext': 'mp4',
 530             'title': 'Rose\'s son Leo was in tears after his dad won gold',
 531             'description': 'Olympic gold medalist Justin Rose gets emotional talking to the impact his win in men\'s golf has already had on his children.',
 532             'timestamp': 1471274964,
 533             'upload_date': '20160815',
 534             'uploader': 'NBCU-SPORTS',
 535         },
 536     }
 537
 538     def _real_extract(self, url):
 539         display_id = self._match_id(url)
 540
 541         webpage = self._download_webpage(url, display_id)
 542
 543         try:
 544             drupal_settings = self._parse_json(self._search_regex(
 545                 r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
 546                 webpage, 'drupal settings'), display_id)
 547
 548             iframe_url = drupal_settings['vod']['iframe_url']
 549             theplatform_url = iframe_url.replace(
 550                 'vplayer.nbcolympics.com', 'player.theplatform.com')
 551         except RegexNotFoundError:
 552             theplatform_url = self._search_regex(
 553                 r"([\"'])embedUrl\1: *([\"'])(?P<embedUrl>.+)\2",
 554                 webpage, 'embedding URL', group="embedUrl")
 555
 556         return {
 557             '_type': 'url_transparent',
 558             'url': theplatform_url,
 559             'ie_key': ThePlatformIE.ie_key(),
 560             'display_id': display_id,
 561         }
 562
 563
 564 class NBCOlympicsStreamIE(AdobePassIE):
 565     IE_NAME = 'nbcolympics:stream'
 566     _VALID_URL = r'https?://stream\.nbcolympics\.com/(?P<id>[0-9a-z-]+)'
 567     _TESTS = [
 568         {
 569             'note': 'Tokenized m3u8 source URL',
 570             'url': 'https://stream.nbcolympics.com/womens-soccer-group-round-11',
 571             'info_dict': {
 572                 'id': '2019740',
 573                 'ext': 'mp4',
 574                 'title': r"re:Women's Group Stage - Netherlands vs\. Brazil [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$",
 575             },
 576             'params': {
 577                 'skip_download': 'm3u8',
 578             },
 579         }, {
 580             'note': 'Plain m3u8 source URL',
 581             'url': 'https://stream.nbcolympics.com/gymnastics-event-finals-mens-floor-pommel-horse-womens-vault-bars',
 582             'info_dict': {
 583                 'id': '2021729',
 584                 'ext': 'mp4',
 585                 'title': r're:Event Finals: M Floor, W Vault, M Pommel, W Uneven Bars [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 586             },
 587             'params': {
 588                 'skip_download': 'm3u8',
 589             },
 590         },
 591     ]
 592
 593     def _real_extract(self, url):
 594         display_id = self._match_id(url)
 595         webpage = self._download_webpage(url, display_id)
 596         pid = self._search_regex(r'pid\s*=\s*(\d+);', webpage, 'pid')
 597
 598         event_config = self._download_json(
 599             f'http://stream.nbcolympics.com/data/event_config_{pid}.json',
 600             pid, 'Downloading event config')['eventConfig']
 601
 602         title = event_config['eventTitle']
 603         is_live = {'live': True, 'replay': False}.get(event_config.get('eventStatus'))
 604
 605         source_url = self._download_json(
 606             f'https://api-leap.nbcsports.com/feeds/assets/{pid}?application=NBCOlympics&platform=desktop&format=nbc-player&env=staging',
 607             pid, 'Downloading leap config'
 608         )['videoSources'][0]['cdnSources']['primary'][0]['sourceUrl']
 609
 610         if event_config.get('cdnToken'):
 611             ap_resource = self._get_mvpd_resource(
 612                 event_config.get('resourceId', 'NBCOlympics'),
 613                 re.sub(r'[^\w\d ]+', '', event_config['eventTitle']), pid,
 614                 event_config.get('ratingId', 'NO VALUE'))
 615             media_token = self._extract_mvpd_auth(url, pid, event_config.get('requestorId', 'NBCOlympics'), ap_resource)
 616
 617             source_url = self._download_json(
 618                 'https://tokens.playmakerservices.com/', pid, 'Retrieving tokenized URL',
 619                 data=json.dumps({
 620                     'application': 'NBCSports',
 621                     'authentication-type': 'adobe-pass',
 622                     'cdn': 'akamai',
 623                     'pid': pid,
 624                     'platform': 'desktop',
 625                     'requestorId': 'NBCOlympics',
 626                     'resourceId': base64.b64encode(ap_resource.encode()).decode(),
 627                     'token': base64.b64encode(media_token.encode()).decode(),
 628                     'url': source_url,
 629                     'version': 'v1',
 630                 }).encode(),
 631             )['akamai'][0]['tokenizedUrl']
 632
 633         formats = self._extract_m3u8_formats(source_url, pid, 'mp4', live=is_live)
 634         for f in formats:
 635             # -http_seekable requires ffmpeg 4.3+ but it doesnt seem possible to
 636             # download with ffmpeg without this option
 637             f['downloader_options'] = {'ffmpeg_args': ['-seekable', '0', '-http_seekable', '0', '-icy', '0']}
 638
 639         return {
 640             'id': pid,
 641             'display_id': display_id,
 642             'title': title,
 643             'formats': formats,
 644             'is_live': is_live,
 645         }
 646
 647
 648 class NBCStationsIE(InfoExtractor):
 649     _DOMAIN_RE = '|'.join(map(re.escape, (
 650         'nbcbayarea', 'nbcboston', 'nbcchicago', 'nbcconnecticut', 'nbcdfw', 'nbclosangeles',
 651         'nbcmiami', 'nbcnewyork', 'nbcphiladelphia', 'nbcsandiego', 'nbcwashington',
 652         'necn', 'telemundo52', 'telemundoarizona', 'telemundochicago', 'telemundonuevainglaterra',
 653     )))
 654     _VALID_URL = rf'https?://(?:www\.)?(?P<site>{_DOMAIN_RE})\.com/(?:[^/?#]+/)*(?P<id>[^/?#]+)/?(?:$|[#?])'
 655
 656     _TESTS = [{
 657         'url': 'https://www.nbclosangeles.com/news/local/large-structure-fire-in-downtown-la-prompts-smoke-odor-advisory/2968618/',
 658         'info_dict': {
 659             'id': '2968618',
 660             'ext': 'mp4',
 661             'title': 'Large Structure Fire in Downtown LA Prompts Smoke Odor Advisory',
 662             'description': 'md5:417ed3c2d91fe9d301e6db7b0942f182',
 663             'timestamp': 1661135892,
 664             'upload_date': '20220822',
 665             'uploader': 'NBC 4',
 666             'channel_id': 'KNBC',
 667             'channel': 'nbclosangeles',
 668         },
 669         'params': {
 670             'skip_download': 'm3u8',
 671         },
 672     }, {
 673         'url': 'https://www.telemundoarizona.com/responde/huracan-complica-reembolso-para-televidente-de-tucson/2247002/',
 674         'info_dict': {
 675             'id': '2247002',
 676             'ext': 'mp4',
 677             'title': 'Huracán complica que televidente de Tucson reciba  reembolso',
 678             'description': 'md5:af298dc73aab74d4fca6abfb12acb6cf',
 679             'timestamp': 1660886507,
 680             'upload_date': '20220819',
 681             'uploader': 'Telemundo Arizona',
 682             'channel_id': 'KTAZ',
 683             'channel': 'telemundoarizona',
 684         },
 685         'params': {
 686             'skip_download': 'm3u8',
 687         },
 688     }]
 689
 690     _RESOLUTIONS = {
 691         '1080': '1920',
 692         '720': '1280',
 693         '540': '960',
 694         '360': '640',
 695         '234': '416',
 696     }
 697
 698     def _real_extract(self, url):
 699         channel, video_id = self._match_valid_url(url).group('site', 'id')
 700         webpage = self._download_webpage(url, video_id)
 701
 702         nbc_data = self._search_json(
 703             r'<script>\s*var\s+nbc\s*=', webpage, 'NBC JSON data', video_id)
 704         pdk_acct = nbc_data.get('pdkAcct') or 'Yh1nAC'
 705         fw_ssid = traverse_obj(nbc_data, ('video', 'fwSSID'))
 706
 707         video_data = self._search_json(
 708             r'data-videos="\[', webpage, 'video data', video_id, default={}, transform_source=unescapeHTML)
 709         video_data.update(self._search_json(
 710             r'data-meta="', webpage, 'metadata', video_id, default={}, transform_source=unescapeHTML))
 711         if not video_data:
 712             raise ExtractorError('No video metadata found in webpage', expected=True)
 713
 714         info, formats, subtitles = {}, [], {}
 715         is_live = int_or_none(video_data.get('mpx_is_livestream')) == 1
 716         query = {
 717             'formats': 'MPEG-DASH none,M3U none,MPEG-DASH none,MPEG4,MP3',
 718             'format': 'SMIL',
 719             'fwsitesection': fw_ssid,
 720             'fwNetworkID': traverse_obj(nbc_data, ('video', 'fwNetworkID'), default='382114'),
 721             'pprofile': 'ots_desktop_html',
 722             'sensitive': 'false',
 723             'w': '1920',
 724             'h': '1080',
 725             'mode': 'LIVE' if is_live else 'on-demand',
 726             'vpaid': 'script',
 727             'schema': '2.0',
 728             'sdk': 'PDK 6.1.3',
 729         }
 730
 731         if is_live:
 732             player_id = traverse_obj(video_data, ((None, ('video', 'meta')), (
 733                 'mpx_m3upid', 'mpx_pid', 'pid_streaming_web_medium')), get_all=False)
 734             info['title'] = f'{channel} livestream'
 735
 736         else:
 737             player_id = traverse_obj(video_data, (
 738                 (None, ('video', 'meta')), ('pid_streaming_web_high', 'mpx_pid')), get_all=False)
 739
 740             date_string = traverse_obj(video_data, 'date_string', 'date_gmt')
 741             if date_string:
 742                 date_string = self._search_regex(
 743                     r'datetime="([^"]+)"', date_string, 'date string', fatal=False)
 744             else:
 745                 date_string = traverse_obj(
 746                     nbc_data, ('dataLayer', 'adobe', ('prop70', 'eVar70', 'eVar59')), get_all=False)
 747
 748             video_url = traverse_obj(video_data, ((None, ('video', 'meta')), 'mp4_url'), get_all=False)
 749             if video_url:
 750                 height = self._search_regex(r'\d+-(\d+)p', url_basename(video_url), 'height', default=None)
 751                 formats.append({
 752                     'url': video_url,
 753                     'ext': 'mp4',
 754                     'width': int_or_none(self._RESOLUTIONS.get(height)),
 755                     'height': int_or_none(height),
 756                     'format_id': 'http-mp4',
 757                 })
 758
 759             info.update({
 760                 'title': video_data.get('title') or traverse_obj(nbc_data, (
 761                     'dataLayer', (None, 'adobe'), ('contenttitle', 'title', 'prop22')), get_all=False),
 762                 'description':
 763                     traverse_obj(video_data, 'summary', 'excerpt', 'video_hero_text')
 764                     or clean_html(traverse_obj(nbc_data, ('dataLayer', 'summary'))),
 765                 'timestamp': unified_timestamp(date_string),
 766             })
 767
 768         smil = None
 769         if player_id and fw_ssid:
 770             smil = self._download_xml(
 771                 f'https://link.theplatform.com/s/{pdk_acct}/{player_id}', video_id,
 772                 note='Downloading SMIL data', query=query, fatal=is_live)
 773         if smil:
 774             manifest_url = xpath_attr(smil, f'.//{{{default_ns}}}video', 'src', fatal=is_live)
 775             subtitles = self._parse_smil_subtitles(smil, default_ns)
 776             fmts, subs = self._extract_m3u8_formats_and_subtitles(
 777                 manifest_url, video_id, 'mp4', m3u8_id='hls', fatal=is_live,
 778                 live=is_live, errnote='No HLS formats found')
 779             formats.extend(fmts)
 780             self._merge_subtitles(subs, target=subtitles)
 781
 782         if not formats:
 783             self.raise_no_formats('No video content found in webpage', expected=True)
 784         elif is_live:
 785             try:
 786                 self._request_webpage(
 787                     HEADRequest(formats[0]['url']), video_id, note='Checking live status')
 788             except ExtractorError:
 789                 raise UserNotLive(video_id=channel)
 790
 791         return {
 792             'id': video_id,
 793             'channel': channel,
 794             'channel_id': nbc_data.get('callLetters'),
 795             'uploader': nbc_data.get('on_air_name'),
 796             'formats': formats,
 797             'subtitles': subtitles,
 798             'is_live': is_live,
 799             **info,
 800         }