yt_dlp/extractor/cbc.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5 import json
   6 import base64
   7 import time
   8
   9 from .common import InfoExtractor
  10 from ..compat import (
  11     compat_str,
  12 )
  13 from ..utils import (
  14     int_or_none,
  15     join_nonempty,
  16     js_to_json,
  17     orderedSet,
  18     smuggle_url,
  19     strip_or_none,
  20     try_get,
  21     ExtractorError,
  22 )
  23
  24
  25 class CBCIE(InfoExtractor):
  26     IE_NAME = 'cbc.ca'
  27     _VALID_URL = r'https?://(?:www\.)?cbc\.ca/(?!player/)(?:[^/]+/)+(?P<id>[^/?#]+)'
  28     _TESTS = [{
  29         # with mediaId
  30         'url': 'http://www.cbc.ca/22minutes/videos/clips-season-23/don-cherry-play-offs',
  31         'md5': '97e24d09672fc4cf56256d6faa6c25bc',
  32         'info_dict': {
  33             'id': '2682904050',
  34             'ext': 'mp4',
  35             'title': 'Don Cherry – All-Stars',
  36             'description': 'Don Cherry has a bee in his bonnet about AHL player John Scott because that guy’s got heart.',
  37             'timestamp': 1454463000,
  38             'upload_date': '20160203',
  39             'uploader': 'CBCC-NEW',
  40         },
  41         'skip': 'Geo-restricted to Canada',
  42     }, {
  43         # with clipId, feed available via tpfeed.cbc.ca and feed.theplatform.com
  44         'url': 'http://www.cbc.ca/22minutes/videos/22-minutes-update/22-minutes-update-episode-4',
  45         'md5': '162adfa070274b144f4fdc3c3b8207db',
  46         'info_dict': {
  47             'id': '2414435309',
  48             'ext': 'mp4',
  49             'title': '22 Minutes Update: What Not To Wear Quebec',
  50             'description': "This week's latest Canadian top political story is What Not To Wear Quebec.",
  51             'upload_date': '20131025',
  52             'uploader': 'CBCC-NEW',
  53             'timestamp': 1382717907,
  54         },
  55         'skip': 'No longer available',
  56     }, {
  57         # with clipId, feed only available via tpfeed.cbc.ca
  58         'url': 'http://www.cbc.ca/archives/entry/1978-robin-williams-freestyles-on-90-minutes-live',
  59         'md5': '0274a90b51a9b4971fe005c63f592f12',
  60         'info_dict': {
  61             'id': '2487345465',
  62             'ext': 'mp4',
  63             'title': 'Robin Williams freestyles on 90 Minutes Live',
  64             'description': 'Wacky American comedian Robin Williams shows off his infamous "freestyle" comedic talents while being interviewed on CBC\'s 90 Minutes Live.',
  65             'upload_date': '19780210',
  66             'uploader': 'CBCC-NEW',
  67             'timestamp': 255977160,
  68         },
  69     }, {
  70         # multiple iframes
  71         'url': 'http://www.cbc.ca/natureofthings/blog/birds-eye-view-from-vancouvers-burrard-street-bridge-how-we-got-the-shot',
  72         'playlist': [{
  73             'md5': '377572d0b49c4ce0c9ad77470e0b96b4',
  74             'info_dict': {
  75                 'id': '2680832926',
  76                 'ext': 'mp4',
  77                 'title': 'An Eagle\'s-Eye View Off Burrard Bridge',
  78                 'description': 'Hercules the eagle flies from Vancouver\'s Burrard Bridge down to a nearby park with a mini-camera strapped to his back.',
  79                 'upload_date': '20160201',
  80                 'timestamp': 1454342820,
  81                 'uploader': 'CBCC-NEW',
  82             },
  83         }, {
  84             'md5': '415a0e3f586113894174dfb31aa5bb1a',
  85             'info_dict': {
  86                 'id': '2658915080',
  87                 'ext': 'mp4',
  88                 'title': 'Fly like an eagle!',
  89                 'description': 'Eagle equipped with a mini camera flies from the world\'s tallest tower',
  90                 'upload_date': '20150315',
  91                 'timestamp': 1426443984,
  92                 'uploader': 'CBCC-NEW',
  93             },
  94         }],
  95         'skip': 'Geo-restricted to Canada',
  96     }, {
  97         # multiple CBC.APP.Caffeine.initInstance(...)
  98         'url': 'http://www.cbc.ca/news/canada/calgary/dog-indoor-exercise-winter-1.3928238',
  99         'info_dict': {
 100             'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
 101             'id': 'dog-indoor-exercise-winter-1.3928238',
 102             'description': 'md5:c18552e41726ee95bd75210d1ca9194c',
 103         },
 104         'playlist_mincount': 6,
 105     }]
 106
 107     @classmethod
 108     def suitable(cls, url):
 109         return False if CBCPlayerIE.suitable(url) else super(CBCIE, cls).suitable(url)
 110
 111     def _extract_player_init(self, player_init, display_id):
 112         player_info = self._parse_json(player_init, display_id, js_to_json)
 113         media_id = player_info.get('mediaId')
 114         if not media_id:
 115             clip_id = player_info['clipId']
 116             feed = self._download_json(
 117                 'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
 118                 clip_id, fatal=False)
 119             if feed:
 120                 media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
 121             if not media_id:
 122                 media_id = self._download_json(
 123                     'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
 124                     clip_id)['entries'][0]['id'].split('/')[-1]
 125         return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
 126
 127     def _real_extract(self, url):
 128         display_id = self._match_id(url)
 129         webpage = self._download_webpage(url, display_id)
 130         title = self._og_search_title(webpage, default=None) or self._html_search_meta(
 131             'twitter:title', webpage, 'title', default=None) or self._html_search_regex(
 132                 r'<title>([^<]+)</title>', webpage, 'title', fatal=False)
 133         entries = [
 134             self._extract_player_init(player_init, display_id)
 135             for player_init in re.findall(r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage)]
 136         media_ids = []
 137         for media_id_re in (
 138                 r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"',
 139                 r'<div[^>]+\bid=["\']player-(\d+)',
 140                 r'guid["\']\s*:\s*["\'](\d+)'):
 141             media_ids.extend(re.findall(media_id_re, webpage))
 142         entries.extend([
 143             self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
 144             for media_id in orderedSet(media_ids)])
 145         return self.playlist_result(
 146             entries, display_id, strip_or_none(title),
 147             self._og_search_description(webpage))
 148
 149
 150 class CBCPlayerIE(InfoExtractor):
 151     IE_NAME = 'cbc.ca:player'
 152     _VALID_URL = r'(?:cbcplayer:|https?://(?:www\.)?cbc\.ca/(?:player/play/|i/caffeine/syndicate/\?mediaId=))(?P<id>\d+)'
 153     _TESTS = [{
 154         'url': 'http://www.cbc.ca/player/play/2683190193',
 155         'md5': '64d25f841ddf4ddb28a235338af32e2c',
 156         'info_dict': {
 157             'id': '2683190193',
 158             'ext': 'mp4',
 159             'title': 'Gerry Runs a Sweat Shop',
 160             'description': 'md5:b457e1c01e8ff408d9d801c1c2cd29b0',
 161             'timestamp': 1455071400,
 162             'upload_date': '20160210',
 163             'uploader': 'CBCC-NEW',
 164         },
 165         'skip': 'Geo-restricted to Canada',
 166     }, {
 167         # Redirected from http://www.cbc.ca/player/AudioMobile/All%20in%20a%20Weekend%20Montreal/ID/2657632011/
 168         'url': 'http://www.cbc.ca/player/play/2657631896',
 169         'md5': 'e5e708c34ae6fca156aafe17c43e8b75',
 170         'info_dict': {
 171             'id': '2657631896',
 172             'ext': 'mp3',
 173             'title': 'CBC Montreal is organizing its first ever community hackathon!',
 174             'description': 'The modern technology we tend to depend on so heavily, is never without it\'s share of hiccups and headaches. Next weekend - CBC Montreal will be getting members of the public for its first Hackathon.',
 175             'timestamp': 1425704400,
 176             'upload_date': '20150307',
 177             'uploader': 'CBCC-NEW',
 178         },
 179     }, {
 180         'url': 'http://www.cbc.ca/player/play/2164402062',
 181         'md5': '33fcd8f6719b9dd60a5e73adcb83b9f6',
 182         'info_dict': {
 183             'id': '2164402062',
 184             'ext': 'mp4',
 185             'title': 'Cancer survivor four times over',
 186             'description': 'Tim Mayer has beaten three different forms of cancer four times in five years.',
 187             'timestamp': 1320410746,
 188             'upload_date': '20111104',
 189             'uploader': 'CBCC-NEW',
 190         },
 191     }]
 192
 193     def _real_extract(self, url):
 194         video_id = self._match_id(url)
 195         return {
 196             '_type': 'url_transparent',
 197             'ie_key': 'ThePlatform',
 198             'url': smuggle_url(
 199                 'http://link.theplatform.com/s/ExhSPC/media/guid/2655402169/%s?mbr=true&formats=MPEG4,FLV,MP3' % video_id, {
 200                     'force_smil_url': True
 201                 }),
 202             'id': video_id,
 203         }
 204
 205
 206 class CBCGemIE(InfoExtractor):
 207     IE_NAME = 'gem.cbc.ca'
 208     _VALID_URL = r'https?://gem\.cbc\.ca/media/(?P<id>[0-9a-z-]+/s[0-9]+[a-z][0-9]+)'
 209     _TESTS = [{
 210         # This is a normal, public, TV show video
 211         'url': 'https://gem.cbc.ca/media/schitts-creek/s06e01',
 212         'md5': '93dbb31c74a8e45b378cf13bd3f6f11e',
 213         'info_dict': {
 214             'id': 'schitts-creek/s06e01',
 215             'ext': 'mp4',
 216             'title': 'Smoke Signals',
 217             'description': 'md5:929868d20021c924020641769eb3e7f1',
 218             'thumbnail': 'https://images.radio-canada.ca/v1/synps-cbc/episode/perso/cbc_schitts_creek_season_06e01_thumbnail_v01.jpg?im=Resize=(Size)',
 219             'duration': 1314,
 220             'categories': ['comedy'],
 221             'series': 'Schitt\'s Creek',
 222             'season': 'Season 6',
 223             'season_number': 6,
 224             'episode': 'Smoke Signals',
 225             'episode_number': 1,
 226             'episode_id': 'schitts-creek/s06e01',
 227         },
 228         'params': {'format': 'bv'},
 229         'skip': 'Geo-restricted to Canada',
 230     }, {
 231         # This video requires an account in the browser, but works fine in yt-dlp
 232         'url': 'https://gem.cbc.ca/media/schitts-creek/s01e01',
 233         'md5': '297a9600f554f2258aed01514226a697',
 234         'info_dict': {
 235             'id': 'schitts-creek/s01e01',
 236             'ext': 'mp4',
 237             'title': 'The Cup Runneth Over',
 238             'description': 'md5:9bca14ea49ab808097530eb05a29e797',
 239             'thumbnail': 'https://images.radio-canada.ca/v1/synps-cbc/episode/perso/cbc_schitts_creek_season_01e01_thumbnail_v01.jpg?im=Resize=(Size)',
 240             'series': 'Schitt\'s Creek',
 241             'season_number': 1,
 242             'season': 'Season 1',
 243             'episode_number': 1,
 244             'episode': 'The Cup Runneth Over',
 245             'episode_id': 'schitts-creek/s01e01',
 246             'duration': 1309,
 247             'categories': ['comedy'],
 248         },
 249         'params': {'format': 'bv'},
 250         'skip': 'Geo-restricted to Canada',
 251     }]
 252
 253     _GEO_COUNTRIES = ['CA']
 254     _TOKEN_API_KEY = '3f4beddd-2061-49b0-ae80-6f1f2ed65b37'
 255     _NETRC_MACHINE = 'cbcgem'
 256     _claims_token = None
 257
 258     def _new_claims_token(self, email, password):
 259         data = json.dumps({
 260             'email': email,
 261             'password': password,
 262         }).encode()
 263         headers = {'content-type': 'application/json'}
 264         query = {'apikey': self._TOKEN_API_KEY}
 265         resp = self._download_json('https://api.loginradius.com/identity/v2/auth/login',
 266                                    None, data=data, headers=headers, query=query)
 267         access_token = resp['access_token']
 268
 269         query = {
 270             'access_token': access_token,
 271             'apikey': self._TOKEN_API_KEY,
 272             'jwtapp': 'jwt',
 273         }
 274         resp = self._download_json('https://cloud-api.loginradius.com/sso/jwt/api/token',
 275                                    None, headers=headers, query=query)
 276         sig = resp['signature']
 277
 278         data = json.dumps({'jwt': sig}).encode()
 279         headers = {'content-type': 'application/json', 'ott-device-type': 'web'}
 280         resp = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/token',
 281                                    None, data=data, headers=headers)
 282         cbc_access_token = resp['accessToken']
 283
 284         headers = {'content-type': 'application/json', 'ott-device-type': 'web', 'ott-access-token': cbc_access_token}
 285         resp = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/profile',
 286                                    None, headers=headers)
 287         return resp['claimsToken']
 288
 289     def _get_claims_token_expiry(self):
 290         # Token is a JWT
 291         # JWT is decoded here and 'exp' field is extracted
 292         # It is a Unix timestamp for when the token expires
 293         b64_data = self._claims_token.split('.')[1]
 294         data = base64.urlsafe_b64decode(b64_data + "==")
 295         return json.loads(data)['exp']
 296
 297     def claims_token_expired(self):
 298         exp = self._get_claims_token_expiry()
 299         if exp - time.time() < 10:
 300             # It will expire in less than 10 seconds, or has already expired
 301             return True
 302         return False
 303
 304     def claims_token_valid(self):
 305         return self._claims_token is not None and not self.claims_token_expired()
 306
 307     def _get_claims_token(self, email, password):
 308         if not self.claims_token_valid():
 309             self._claims_token = self._new_claims_token(email, password)
 310             self._downloader.cache.store(self._NETRC_MACHINE, 'claims_token', self._claims_token)
 311         return self._claims_token
 312
 313     def _real_initialize(self):
 314         if self.claims_token_valid():
 315             return
 316         self._claims_token = self._downloader.cache.load(self._NETRC_MACHINE, 'claims_token')
 317
 318     def _find_secret_formats(self, formats, video_id):
 319         """ Find a valid video url and convert it to the secret variant """
 320         base_format = next((f for f in formats if f.get('vcodec') != 'none'), None)
 321         if not base_format:
 322             return
 323
 324         base_url = re.sub(r'(Manifest\(.*?),filter=[\w-]+(.*?\))', r'\1\2', base_format['url'])
 325         url = re.sub(r'(Manifest\(.*?),format=[\w-]+(.*?\))', r'\1\2', base_url)
 326
 327         secret_xml = self._download_xml(url, video_id, note='Downloading secret XML', fatal=False)
 328         if not secret_xml:
 329             return
 330
 331         for child in secret_xml:
 332             if child.attrib.get('Type') != 'video':
 333                 continue
 334             for video_quality in child:
 335                 bitrate = int_or_none(video_quality.attrib.get('Bitrate'))
 336                 if not bitrate or 'Index' not in video_quality.attrib:
 337                     continue
 338                 height = int_or_none(video_quality.attrib.get('MaxHeight'))
 339
 340                 yield {
 341                     **base_format,
 342                     'format_id': join_nonempty('sec', height),
 343                     'url': re.sub(r'(QualityLevels\()\d+(\))', fr'\1{bitrate}\2', base_url),
 344                     'width': int_or_none(video_quality.attrib.get('MaxWidth')),
 345                     'tbr': bitrate / 1000.0,
 346                     'height': height,
 347                 }
 348
 349     def _real_extract(self, url):
 350         video_id = self._match_id(url)
 351         video_info = self._download_json('https://services.radio-canada.ca/ott/cbc-api/v2/assets/' + video_id, video_id)
 352
 353         email, password = self._get_login_info()
 354         if email and password:
 355             claims_token = self._get_claims_token(email, password)
 356             headers = {'x-claims-token': claims_token}
 357         else:
 358             headers = {}
 359         m3u8_info = self._download_json(video_info['playSession']['url'], video_id, headers=headers)
 360         m3u8_url = m3u8_info.get('url')
 361
 362         if m3u8_info.get('errorCode') == 1:
 363             self.raise_geo_restricted(countries=['CA'])
 364         elif m3u8_info.get('errorCode') == 35:
 365             self.raise_login_required(method='password')
 366         elif m3u8_info.get('errorCode') != 0:
 367             raise ExtractorError(f'{self.IE_NAME} said: {m3u8_info.get("errorCode")} - {m3u8_info.get("message")}')
 368
 369         formats = self._extract_m3u8_formats(m3u8_url, video_id, m3u8_id='hls')
 370         self._remove_duplicate_formats(formats)
 371         formats.extend(self._find_secret_formats(formats, video_id))
 372
 373         for format in formats:
 374             if format.get('vcodec') == 'none':
 375                 if format.get('ext') is None:
 376                     format['ext'] = 'm4a'
 377                 if format.get('acodec') is None:
 378                     format['acodec'] = 'mp4a.40.2'
 379
 380                 # Put described audio at the beginning of the list, so that it
 381                 # isn't chosen by default, as most people won't want it.
 382                 if 'descriptive' in format['format_id'].lower():
 383                     format['preference'] = -2
 384
 385         self._sort_formats(formats)
 386
 387         return {
 388             'id': video_id,
 389             'title': video_info['title'],
 390             'description': video_info.get('description'),
 391             'thumbnail': video_info.get('image'),
 392             'series': video_info.get('series'),
 393             'season_number': video_info.get('season'),
 394             'season': f'Season {video_info.get("season")}',
 395             'episode_number': video_info.get('episode'),
 396             'episode': video_info.get('title'),
 397             'episode_id': video_id,
 398             'duration': video_info.get('duration'),
 399             'categories': [video_info.get('category')],
 400             'formats': formats,
 401             'release_timestamp': video_info.get('airDate'),
 402             'timestamp': video_info.get('availableDate'),
 403         }
 404
 405
 406 class CBCGemPlaylistIE(InfoExtractor):
 407     IE_NAME = 'gem.cbc.ca:playlist'
 408     _VALID_URL = r'https?://gem\.cbc\.ca/media/(?P<id>(?P<show>[0-9a-z-]+)/s(?P<season>[0-9]+))/?(?:[?#]|$)'
 409     _TESTS = [{
 410         # TV show playlist, all public videos
 411         'url': 'https://gem.cbc.ca/media/schitts-creek/s06',
 412         'playlist_count': 16,
 413         'info_dict': {
 414             'id': 'schitts-creek/s06',
 415             'title': 'Season 6',
 416             'description': 'md5:6a92104a56cbeb5818cc47884d4326a2',
 417         },
 418     }]
 419     _API_BASE = 'https://services.radio-canada.ca/ott/cbc-api/v2/shows/'
 420
 421     def _real_extract(self, url):
 422         match = self._match_valid_url(url)
 423         season_id = match.group('id')
 424         show = match.group('show')
 425         show_info = self._download_json(self._API_BASE + show, season_id)
 426         season = int(match.group('season'))
 427
 428         season_info = next((s for s in show_info['seasons'] if s.get('season') == season), None)
 429
 430         if season_info is None:
 431             raise ExtractorError(f'Couldn\'t find season {season} of {show}')
 432
 433         episodes = []
 434         for episode in season_info['assets']:
 435             episodes.append({
 436                 '_type': 'url_transparent',
 437                 'ie_key': 'CBCGem',
 438                 'url': 'https://gem.cbc.ca/media/' + episode['id'],
 439                 'id': episode['id'],
 440                 'title': episode.get('title'),
 441                 'description': episode.get('description'),
 442                 'thumbnail': episode.get('image'),
 443                 'series': episode.get('series'),
 444                 'season_number': episode.get('season'),
 445                 'season': season_info['title'],
 446                 'season_id': season_info.get('id'),
 447                 'episode_number': episode.get('episode'),
 448                 'episode': episode.get('title'),
 449                 'episode_id': episode['id'],
 450                 'duration': episode.get('duration'),
 451                 'categories': [episode.get('category')],
 452             })
 453
 454         thumbnail = None
 455         tn_uri = season_info.get('image')
 456         # the-national was observed to use a "data:image/png;base64"
 457         # URI for their 'image' value. The image was 1x1, and is
 458         # probably just a placeholder, so it is ignored.
 459         if tn_uri is not None and not tn_uri.startswith('data:'):
 460             thumbnail = tn_uri
 461
 462         return {
 463             '_type': 'playlist',
 464             'entries': episodes,
 465             'id': season_id,
 466             'title': season_info['title'],
 467             'description': season_info.get('description'),
 468             'thumbnail': thumbnail,
 469             'series': show_info.get('title'),
 470             'season_number': season_info.get('season'),
 471             'season': season_info['title'],
 472         }
 473
 474
 475 class CBCGemLiveIE(InfoExtractor):
 476     IE_NAME = 'gem.cbc.ca:live'
 477     _VALID_URL = r'https?://gem\.cbc\.ca/live/(?P<id>\d+)'
 478     _TEST = {
 479         'url': 'https://gem.cbc.ca/live/920604739687',
 480         'info_dict': {
 481             'title': 'Ottawa',
 482             'description': 'The live TV channel and local programming from Ottawa',
 483             'thumbnail': 'https://thumbnails.cbc.ca/maven_legacy/thumbnails/CBC_OTT_VMS/Live_Channel_Static_Images/Ottawa_2880x1620.jpg',
 484             'is_live': True,
 485             'id': 'AyqZwxRqh8EH',
 486             'ext': 'mp4',
 487             'timestamp': 1492106160,
 488             'upload_date': '20170413',
 489             'uploader': 'CBCC-NEW',
 490         },
 491         'skip': 'Live might have ended',
 492     }
 493
 494     # It's unclear where the chars at the end come from, but they appear to be
 495     # constant. Might need updating in the future.
 496     # There are two URLs, some livestreams are in one, and some
 497     # in the other. The JSON schema is the same for both.
 498     _API_URLS = ['https://tpfeed.cbc.ca/f/ExhSPC/t_t3UKJR6MAT', 'https://tpfeed.cbc.ca/f/ExhSPC/FNiv9xQx_BnT']
 499
 500     def _real_extract(self, url):
 501         video_id = self._match_id(url)
 502
 503         for api_url in self._API_URLS:
 504             video_info = next((
 505                 stream for stream in self._download_json(api_url, video_id)['entries']
 506                 if stream.get('guid') == video_id), None)
 507             if video_info:
 508                 break
 509         else:
 510             raise ExtractorError('Couldn\'t find video metadata, maybe this livestream is now offline', expected=True)
 511
 512         return {
 513             '_type': 'url_transparent',
 514             'ie_key': 'ThePlatform',
 515             'url': video_info['content'][0]['url'],
 516             'id': video_id,
 517             'title': video_info.get('title'),
 518             'description': video_info.get('description'),
 519             'tags': try_get(video_info, lambda x: x['keywords'].split(', ')),
 520             'thumbnail': video_info.get('cbc$staticImage'),
 521             'is_live': True,
 522         }