yt_dlp/extractor/go.py

   1 import re
   2
   3 from .adobepass import AdobePassIE
   4 from ..utils import (
   5     ExtractorError,
   6     determine_ext,
   7     int_or_none,
   8     parse_age_limit,
   9     remove_end,
  10     remove_start,
  11     traverse_obj,
  12     try_get,
  13     unified_timestamp,
  14     urlencode_postdata,
  15 )
  16
  17
  18 class GoIE(AdobePassIE):
  19     _SITE_INFO = {
  20         'abc': {
  21             'brand': '001',
  22             'requestor_id': 'ABC',
  23         },
  24         'freeform': {
  25             'brand': '002',
  26             'requestor_id': 'ABCFamily',
  27         },
  28         'watchdisneychannel': {
  29             'brand': '004',
  30             'resource_id': 'Disney',
  31         },
  32         'watchdisneyjunior': {
  33             'brand': '008',
  34             'resource_id': 'DisneyJunior',
  35         },
  36         'watchdisneyxd': {
  37             'brand': '009',
  38             'resource_id': 'DisneyXD',
  39         },
  40         'disneynow': {
  41             'brand': '011',
  42             'resource_id': 'Disney',
  43         },
  44         'fxnow.fxnetworks': {
  45             'brand': '025',
  46             'requestor_id': 'dtci',
  47         },
  48     }
  49     _VALID_URL = r'''(?x)
  50                     https?://
  51                         (?P<sub_domain>
  52                             (?:{}\.)?go|fxnow\.fxnetworks|
  53                             (?:www\.)?(?:abc|freeform|disneynow)
  54                         )\.com/
  55                         (?:
  56                             (?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)|
  57                             (?:[^/]+/)*(?P<display_id>[^/?\#]+)
  58                         )
  59                     '''.format(r'\.|'.join(list(_SITE_INFO.keys())))
  60     _TESTS = [{
  61         'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
  62         'info_dict': {
  63             'id': 'VDKA3807643',
  64             'ext': 'mp4',
  65             'title': 'The Traitor in the White House',
  66             'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
  67         },
  68         'params': {
  69             # m3u8 download
  70             'skip_download': True,
  71         },
  72         'skip': 'This content is no longer available.',
  73     }, {
  74         'url': 'https://disneynow.com/shows/big-hero-6-the-series',
  75         'info_dict': {
  76             'title': 'Doraemon',
  77             'id': 'SH55574025',
  78         },
  79         'playlist_mincount': 51,
  80     }, {
  81         'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood',
  82         'info_dict': {
  83             'id': 'VDKA3609139',
  84             'title': 'This Guilty Blood',
  85             'description': 'md5:f18e79ad1c613798d95fdabfe96cd292',
  86             'age_limit': 14,
  87             'episode': 'Episode 1',
  88             'upload_date': '20170102',
  89             'season': 'Season 2',
  90             'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abcf/Shadowhunters/video/201/ae5f75608d86bf88aa4f9f4aa76ab1b7/579x325-Q100_ae5f75608d86bf88aa4f9f4aa76ab1b7.jpg',
  91             'duration': 2544,
  92             'season_number': 2,
  93             'series': 'Shadowhunters',
  94             'episode_number': 1,
  95             'timestamp': 1483387200,
  96             'ext': 'mp4',
  97         },
  98         'params': {
  99             'geo_bypass_ip_block': '3.244.239.0/24',
 100             # m3u8 download
 101             'skip_download': True,
 102         },
 103     }, {
 104         'url': 'https://abc.com/shows/the-rookie/episode-guide/season-04/12-the-knock',
 105         'info_dict': {
 106             'id': 'VDKA26050359',
 107             'title': 'The Knock',
 108             'description': 'md5:0c2947e3ada4c31f28296db7db14aa64',
 109             'age_limit': 14,
 110             'ext': 'mp4',
 111             'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abc/TheRookie/video/412/daf830d06e83b11eaf5c0a299d993ae3/1556x876-Q75_daf830d06e83b11eaf5c0a299d993ae3.jpg',
 112             'episode': 'Episode 12',
 113             'season_number': 4,
 114             'season': 'Season 4',
 115             'timestamp': 1642975200,
 116             'episode_number': 12,
 117             'upload_date': '20220123',
 118             'series': 'The Rookie',
 119             'duration': 2572,
 120         },
 121         'params': {
 122             'geo_bypass_ip_block': '3.244.239.0/24',
 123             # m3u8 download
 124             'skip_download': True,
 125         },
 126     }, {
 127         'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841',
 128         'info_dict': {
 129             'id': 'VDKA12782841',
 130             'title': 'First Look: Better Things - Season 2',
 131             'description': 'md5:fa73584a95761c605d9d54904e35b407',
 132             'ext': 'mp4',
 133             'age_limit': 14,
 134             'upload_date': '20170825',
 135             'duration': 161,
 136             'series': 'Better Things',
 137             'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/fx/BetterThings/video/12782841/b6b05e58264121cc2c98811318e6d507/1556x876-Q75_b6b05e58264121cc2c98811318e6d507.jpg',
 138             'timestamp': 1503661074,
 139         },
 140         'params': {
 141             'geo_bypass_ip_block': '3.244.239.0/24',
 142             # m3u8 download
 143             'skip_download': True,
 144         },
 145     }, {
 146         'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
 147         'only_matching': True,
 148     }, {
 149         'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
 150         'only_matching': True,
 151     }, {
 152         # brand 004
 153         'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915',
 154         'only_matching': True,
 155     }, {
 156         # brand 008
 157         'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
 158         'only_matching': True,
 159     }, {
 160         'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
 161         'only_matching': True,
 162     }, {
 163         'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner',
 164         'only_matching': True,
 165     }]
 166
 167     def _extract_videos(self, brand, video_id='-1', show_id='-1'):
 168         display_id = video_id if video_id != '-1' else show_id
 169         return self._download_json(
 170             f'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/{brand}/001/-1/{show_id}/-1/{video_id}/-1/-1.json',
 171             display_id)['video']
 172
 173     def _real_extract(self, url):
 174         mobj = self._match_valid_url(url)
 175         sub_domain = remove_start(remove_end(mobj.group('sub_domain') or '', '.go'), 'www.')
 176         video_id, display_id = mobj.group('id', 'display_id')
 177         site_info = self._SITE_INFO.get(sub_domain, {})
 178         brand = site_info.get('brand')
 179         if not video_id or not site_info:
 180             webpage = self._download_webpage(url, display_id or video_id)
 181             data = self._parse_json(
 182                 self._search_regex(
 183                     r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage,
 184                     'data', default='{}'),
 185                 display_id or video_id, fatal=False)
 186             # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot
 187             layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict)
 188             video_id = None
 189             if layout:
 190                 video_id = try_get(
 191                     layout,
 192                     (lambda x: x['videoid'], lambda x: x['video']['id']),
 193                     str)
 194             if not video_id:
 195                 video_id = self._search_regex(
 196                     (
 197                         # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
 198                         # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
 199                         r'data-video-id=["\']*(VDKA\w+)',
 200                         # page.analytics.videoIdCode
 201                         r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)',
 202                         # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet
 203                         r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)',
 204                     ), webpage, 'video id', default=video_id)
 205             if not site_info:
 206                 brand = self._search_regex(
 207                     (r'data-brand=\s*["\']\s*(\d+)',
 208                      r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand',
 209                     default='004')
 210                 site_info = next(
 211                     si for _, si in self._SITE_INFO.items()
 212                     if si.get('brand') == brand)
 213             if not video_id:
 214                 # show extraction works for Disney, DisneyJunior and DisneyXD
 215                 # ABC and Freeform has different layout
 216                 show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
 217                 videos = self._extract_videos(brand, show_id=show_id)
 218                 show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
 219                 entries = []
 220                 for video in videos:
 221                     entries.append(self.url_result(
 222                         video['url'], 'Go', video.get('id'), video.get('title')))
 223                 entries.reverse()
 224                 return self.playlist_result(entries, show_id, show_title)
 225         video_data = self._extract_videos(brand, video_id)[0]
 226         video_id = video_data['id']
 227         title = video_data['title']
 228
 229         formats = []
 230         subtitles = {}
 231         for asset in video_data.get('assets', {}).get('asset', []):
 232             asset_url = asset.get('value')
 233             if not asset_url:
 234                 continue
 235             format_id = asset.get('format')
 236             ext = determine_ext(asset_url)
 237             if ext == 'm3u8':
 238                 video_type = video_data.get('type')
 239                 data = {
 240                     'video_id': video_data['id'],
 241                     'video_type': video_type,
 242                     'brand': brand,
 243                     'device': '001',
 244                 }
 245                 if video_data.get('accesslevel') == '1':
 246                     requestor_id = site_info.get('requestor_id', 'DisneyChannels')
 247                     resource = site_info.get('resource_id') or self._get_mvpd_resource(
 248                         requestor_id, title, video_id, None)
 249                     auth = self._extract_mvpd_auth(
 250                         url, video_id, requestor_id, resource)
 251                     data.update({
 252                         'token': auth,
 253                         'token_type': 'ap',
 254                         'adobe_requestor_id': requestor_id,
 255                     })
 256                 else:
 257                     self._initialize_geo_bypass({'countries': ['US']})
 258                 entitlement = self._download_json(
 259                     'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
 260                     video_id, data=urlencode_postdata(data))
 261                 errors = entitlement.get('errors', {}).get('errors', [])
 262                 if errors:
 263                     for error in errors:
 264                         if error.get('code') == 1002:
 265                             self.raise_geo_restricted(
 266                                 error['message'], countries=['US'])
 267                     error_message = ', '.join([error['message'] for error in errors])
 268                     raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True)
 269                 asset_url += '?' + entitlement['uplynkData']['sessionKey']
 270                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
 271                     asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False)
 272                 formats.extend(fmts)
 273                 self._merge_subtitles(subs, target=subtitles)
 274             else:
 275                 f = {
 276                     'format_id': format_id,
 277                     'url': asset_url,
 278                     'ext': ext,
 279                 }
 280                 if re.search(r'(?:/mp4/source/|_source\.mp4)', asset_url):
 281                     f.update({
 282                         'format_id': (f'{format_id}-' if format_id else '') + 'SOURCE',
 283                         'quality': 1,
 284                     })
 285                 else:
 286                     mobj = re.search(r'/(\d+)x(\d+)/', asset_url)
 287                     if mobj:
 288                         height = int(mobj.group(2))
 289                         f.update({
 290                             'format_id': (f'{format_id}-' if format_id else '') + f'{height}P',
 291                             'width': int(mobj.group(1)),
 292                             'height': height,
 293                         })
 294                 formats.append(f)
 295
 296         for cc in video_data.get('closedcaption', {}).get('src', []):
 297             cc_url = cc.get('value')
 298             if not cc_url:
 299                 continue
 300             ext = determine_ext(cc_url)
 301             if ext == 'xml':
 302                 ext = 'ttml'
 303             subtitles.setdefault(cc.get('lang'), []).append({
 304                 'url': cc_url,
 305                 'ext': ext,
 306             })
 307
 308         thumbnails = []
 309         for thumbnail in video_data.get('thumbnails', {}).get('thumbnail', []):
 310             thumbnail_url = thumbnail.get('value')
 311             if not thumbnail_url:
 312                 continue
 313             thumbnails.append({
 314                 'url': thumbnail_url,
 315                 'width': int_or_none(thumbnail.get('width')),
 316                 'height': int_or_none(thumbnail.get('height')),
 317             })
 318
 319         return {
 320             'id': video_id,
 321             'title': title,
 322             'description': video_data.get('longdescription') or video_data.get('description'),
 323             'duration': int_or_none(video_data.get('duration', {}).get('value'), 1000),
 324             'age_limit': parse_age_limit(video_data.get('tvrating', {}).get('rating')),
 325             'episode_number': int_or_none(video_data.get('episodenumber')),
 326             'series': video_data.get('show', {}).get('title'),
 327             'season_number': int_or_none(video_data.get('season', {}).get('num')),
 328             'thumbnails': thumbnails,
 329             'formats': formats,
 330             'subtitles': subtitles,
 331             'timestamp': unified_timestamp(traverse_obj(video_data, ('airdates', 'airdate', 0))),
 332         }