[yt-dlp.git] / yt_dlp / extractor / go.py

# coding: utf-8
from __future__ import unicode_literals

import re

from .adobepass import AdobePassIE
from ..utils import (
    int_or_none,
    determine_ext,
    parse_age_limit,
    urlencode_postdata,
    ExtractorError,
)


class GoIE(AdobePassIE):
    _SITE_INFO = {
        'abc': {
            'brand': '001',
            'requestor_id': 'ABC',
        },
        'freeform': {
            'brand': '002',
            'requestor_id': 'ABCFamily',
        },
        'watchdisneychannel': {
            'brand': '004',
            'resource_id': 'Disney',
        },
        'watchdisneyjunior': {
            'brand': '008',
            'resource_id': 'DisneyJunior',
        },
        'watchdisneyxd': {
            'brand': '009',
            'resource_id': 'DisneyXD',
        },
        'disneynow': {
            'brand': '011',
            'resource_id': 'Disney',
        },
        'fxnow.fxnetworks': {
            'brand': '025',
            'requestor_id': 'dtci',
        },
    }
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            (?:(?P<sub_domain>%s)\.)?go|
                            (?P<sub_domain_2>abc|freeform|disneynow|fxnow\.fxnetworks)
                        )\.com/
                        (?:
                            (?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)|
                            (?:[^/]+/)*(?P<display_id>[^/?\#]+)
                        )
                    ''' % '|'.join(list(_SITE_INFO.keys()))
    _TESTS = [{
        'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
        'info_dict': {
            'id': 'VDKA3807643',
            'ext': 'mp4',
            'title': 'The Traitor in the White House',
            'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
        'skip': 'This content is no longer available.',
    }, {
        'url': 'http://watchdisneyxd.go.com/doraemon',
        'info_dict': {
            'title': 'Doraemon',
            'id': 'SH55574025',
        },
        'playlist_mincount': 51,
    }, {
        'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood',
        'info_dict': {
            'id': 'VDKA3609139',
            'ext': 'mp4',
            'title': 'This Guilty Blood',
            'description': 'md5:f18e79ad1c613798d95fdabfe96cd292',
            'age_limit': 14,
        },
        'params': {
            'geo_bypass_ip_block': '3.244.239.0/24',
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet',
        'info_dict': {
            'id': 'VDKA13435179',
            'ext': 'mp4',
            'title': 'The Bet',
            'description': 'md5:c66de8ba2e92c6c5c113c3ade84ab404',
            'age_limit': 14,
        },
        'params': {
            'geo_bypass_ip_block': '3.244.239.0/24',
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841',
        'info_dict': {
            'id': 'VDKA12782841',
            'ext': 'mp4',
            'title': 'First Look: Better Things - Season 2',
            'description': 'md5:fa73584a95761c605d9d54904e35b407',
        },
        'params': {
            'geo_bypass_ip_block': '3.244.239.0/24',
            # m3u8 download
            'skip_download': True,
        },
    }, {
        'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
        'only_matching': True,
    }, {
        'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
        'only_matching': True,
    }, {
        # brand 004
        'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915',
        'only_matching': True,
    }, {
        # brand 008
        'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
        'only_matching': True,
    }, {
        'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
        'only_matching': True,
    }]

    def _extract_videos(self, brand, video_id='-1', show_id='-1'):
        display_id = video_id if video_id != '-1' else show_id
        return self._download_json(
            'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/%s/-1/%s/-1/-1.json' % (brand, show_id, video_id),
            display_id)['video']

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        sub_domain = mobj.group('sub_domain') or mobj.group('sub_domain_2')
        video_id, display_id = mobj.group('id', 'display_id')
        site_info = self._SITE_INFO.get(sub_domain, {})
        brand = site_info.get('brand')
        if not video_id or not site_info:
            webpage = self._download_webpage(url, display_id or video_id)
            video_id = self._search_regex(
                (
                    # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
                    # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
                    r'data-video-id=["\']*(VDKA\w+)',
                    # https://github.com/ytdl-org/youtube-dl/pull/25216/files
                    #  The following is based on the pull request on the line above. Changed the ABC.com URL to a show available now.
                    # https://abc.com/shows/the-rookie/episode-guide/season-02/19-the-q-word
                    r'\bvideoIdCode["\']\s*:\s*["\'](vdka\w+)',
                    # Deprecated  fallback pattern
                    r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)'
                ), webpage, 'video id', default=video_id)
            if not site_info:
                brand = self._search_regex(
                    (r'data-brand=\s*["\']\s*(\d+)',
                     r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand',
                    default='004')
                site_info = next(
                    si for _, si in self._SITE_INFO.items()
                    if si.get('brand') == brand)
            if not video_id:
                # show extraction works for Disney, DisneyJunior and DisneyXD
                # ABC and Freeform has different layout
                show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
                videos = self._extract_videos(brand, show_id=show_id)
                show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
                entries = []
                for video in videos:
                    entries.append(self.url_result(
                        video['url'], 'Go', video.get('id'), video.get('title')))
                entries.reverse()
                return self.playlist_result(entries, show_id, show_title)
        video_data = self._extract_videos(brand, video_id)[0]
        video_id = video_data['id']
        title = video_data['title']

        formats = []
        for asset in video_data.get('assets', {}).get('asset', []):
            asset_url = asset.get('value')
            if not asset_url:
                continue
            format_id = asset.get('format')
            ext = determine_ext(asset_url)
            if ext == 'm3u8':
                video_type = video_data.get('type')
                data = {
                    'video_id': video_data['id'],
                    'video_type': video_type,
                    'brand': brand,
                    'device': '001',
                }
                if video_data.get('accesslevel') == '1':
                    requestor_id = site_info.get('requestor_id', 'DisneyChannels')
                    resource = site_info.get('resource_id') or self._get_mvpd_resource(
                        requestor_id, title, video_id, None)
                    auth = self._extract_mvpd_auth(
                        url, video_id, requestor_id, resource)
                    data.update({
                        'token': auth,
                        'token_type': 'ap',
                        'adobe_requestor_id': requestor_id,
                    })
                else:
                    self._initialize_geo_bypass({'countries': ['US']})
                entitlement = self._download_json(
                    'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
                    video_id, data=urlencode_postdata(data))
                errors = entitlement.get('errors', {}).get('errors', [])
                if errors:
                    for error in errors:
                        if error.get('code') == 1002:
                            self.raise_geo_restricted(
                                error['message'], countries=['US'])
                    error_message = ', '.join([error['message'] for error in errors])
                    raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
                asset_url += '?' + entitlement['uplynkData']['sessionKey']
                formats.extend(self._extract_m3u8_formats(
                    asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False))
            else:
                f = {
                    'format_id': format_id,
                    'url': asset_url,
                    'ext': ext,
                }
                if re.search(r'(?:/mp4/source/|_source\.mp4)', asset_url):
                    f.update({
                        'format_id': ('%s-' % format_id if format_id else '') + 'SOURCE',
                        'quality': 1,
                    })
                else:
                    mobj = re.search(r'/(\d+)x(\d+)/', asset_url)
                    if mobj:
                        height = int(mobj.group(2))
                        f.update({
                            'format_id': ('%s-' % format_id if format_id else '') + '%dP' % height,
                            'width': int(mobj.group(1)),
                            'height': height,
                        })
                formats.append(f)
        self._sort_formats(formats)

        subtitles = {}
        for cc in video_data.get('closedcaption', {}).get('src', []):
            cc_url = cc.get('value')
            if not cc_url:
                continue
            ext = determine_ext(cc_url)
            if ext == 'xml':
                ext = 'ttml'
            subtitles.setdefault(cc.get('lang'), []).append({
                'url': cc_url,
                'ext': ext,
            })

        thumbnails = []
        for thumbnail in video_data.get('thumbnails', {}).get('thumbnail', []):
            thumbnail_url = thumbnail.get('value')
            if not thumbnail_url:
                continue
            thumbnails.append({
                'url': thumbnail_url,
                'width': int_or_none(thumbnail.get('width')),
                'height': int_or_none(thumbnail.get('height')),
            })

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('longdescription') or video_data.get('description'),
            'duration': int_or_none(video_data.get('duration', {}).get('value'), 1000),
            'age_limit': parse_age_limit(video_data.get('tvrating', {}).get('rating')),
            'episode_number': int_or_none(video_data.get('episodenumber')),
            'series': video_data.get('show', {}).get('title'),
            'season_number': int_or_none(video_data.get('season', {}).get('num')),
            'thumbnails': thumbnails,
            'formats': formats,
            'subtitles': subtitles,
        }
Commit	Line	Data
2c3e0af9 RA	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
	4	import re
	5
ae8d5a5c	6	from .adobepass import AdobePassIE
2c3e0af9 RA	7	from ..utils import (
	8	int_or_none,
	9	determine_ext,
	10	parse_age_limit,
014b7e6b RA	11	urlencode_postdata,
014b7e6b RA	12	ExtractorError,
2c3e0af9 RA	13	)
	14
	15
ae8d5a5c RA	16	class GoIE(AdobePassIE):
	17	_SITE_INFO = {
	18	'abc': {
	19	'brand': '001',
	20	'requestor_id': 'ABC',
	21	},
	22	'freeform': {
	23	'brand': '002',
	24	'requestor_id': 'ABCFamily',
	25	},
	26	'watchdisneychannel': {
	27	'brand': '004',
118afcf5	28	'resource_id': 'Disney',
ae8d5a5c RA	29	},
	30	'watchdisneyjunior': {
	31	'brand': '008',
118afcf5	32	'resource_id': 'DisneyJunior',
ae8d5a5c RA	33	},
	34	'watchdisneyxd': {
	35	'brand': '009',
118afcf5	36	'resource_id': 'DisneyXD',
a30c2f40 S	37	},
	38	'disneynow': {
	39	'brand': '011',
	40	'resource_id': 'Disney',
29f7c58a	41	},
	42	'fxnow.fxnetworks': {
	43	'brand': '025',
	44	'requestor_id': 'dtci',
	45	},
2c3e0af9	46	}
aef9f87e S	47	_VALID_URL = r'''(?x)
	48	https?://
	49	(?:
	50	(?:(?P<sub_domain>%s)\.)?go\|
29f7c58a	51	(?P<sub_domain_2>abc\|freeform\|disneynow\|fxnow\.fxnetworks)
aef9f87e S	52	)\.com/
	53	(?:
	54	(?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)\|
	55	(?:[^/]+/)*(?P<display_id>[^/?\#]+)
	56	)
	57	''' % '\|'.join(list(_SITE_INFO.keys()))
2c3e0af9	58	_TESTS = [{
bf2a5555	59	'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
2c3e0af9	60	'info_dict': {
bf2a5555	61	'id': 'VDKA3807643',
2c3e0af9	62	'ext': 'mp4',
bf2a5555 RA	63	'title': 'The Traitor in the White House',
bf2a5555 RA	64	'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
2c3e0af9 RA	65	},
	66	'params': {
	67	# m3u8 download
	68	'skip_download': True,
	69	},
dd90a21c	70	'skip': 'This content is no longer available.',
2c3e0af9	71	}, {
bf2a5555 RA	72	'url': 'http://watchdisneyxd.go.com/doraemon',
	73	'info_dict': {
	74	'title': 'Doraemon',
	75	'id': 'SH55574025',
	76	},
	77	'playlist_mincount': 51,
dd90a21c S	78	}, {
	79	'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood',
	80	'info_dict': {
	81	'id': 'VDKA3609139',
	82	'ext': 'mp4',
	83	'title': 'This Guilty Blood',
	84	'description': 'md5:f18e79ad1c613798d95fdabfe96cd292',
	85	'age_limit': 14,
	86	},
	87	'params': {
	88	'geo_bypass_ip_block': '3.244.239.0/24',
	89	# m3u8 download
	90	'skip_download': True,
	91	},
	92	}, {
	93	'url': 'https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet',
	94	'info_dict': {
	95	'id': 'VDKA13435179',
	96	'ext': 'mp4',
	97	'title': 'The Bet',
	98	'description': 'md5:c66de8ba2e92c6c5c113c3ade84ab404',
	99	'age_limit': 14,
	100	},
	101	'params': {
	102	'geo_bypass_ip_block': '3.244.239.0/24',
	103	# m3u8 download
	104	'skip_download': True,
	105	},
29f7c58a	106	}, {
	107	'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841',
	108	'info_dict': {
	109	'id': 'VDKA12782841',
	110	'ext': 'mp4',
	111	'title': 'First Look: Better Things - Season 2',
	112	'description': 'md5:fa73584a95761c605d9d54904e35b407',
	113	},
	114	'params': {
	115	'geo_bypass_ip_block': '3.244.239.0/24',
	116	# m3u8 download
	117	'skip_download': True,
	118	},
692fa200 S	119	}, {
	120	'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
	121	'only_matching': True,
	122	}, {
	123	'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
	124	'only_matching': True,
52007de8 S	125	}, {
	126	# brand 004
	127	'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915',
	128	'only_matching': True,
	129	}, {
	130	# brand 008
	131	'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
	132	'only_matching': True,
4f71473e S	133	}, {
	134	'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
	135	'only_matching': True,
2c3e0af9 RA	136	}]
2c3e0af9 RA	137
bf2a5555 RA	138	def _extract_videos(self, brand, video_id='-1', show_id='-1'):
	139	display_id = video_id if video_id != '-1' else show_id
	140	return self._download_json(
	141	'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/%s/001/-1/%s/-1/%s/-1/-1.json' % (brand, show_id, video_id),
	142	display_id)['video']
	143
2c3e0af9	144	def _real_extract(self, url):
a30c2f40 S	145	mobj = re.match(self._VALID_URL, url)
	146	sub_domain = mobj.group('sub_domain') or mobj.group('sub_domain_2')
	147	video_id, display_id = mobj.group('id', 'display_id')
52007de8 S	148	site_info = self._SITE_INFO.get(sub_domain, {})
	149	brand = site_info.get('brand')
	150	if not video_id or not site_info:
	151	webpage = self._download_webpage(url, display_id or video_id)
c54c01f8	152	video_id = self._search_regex(
dd90a21c S	153	(
	154	# There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
	155	# from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
	156	r'data-video-id=["\']*(VDKA\w+)',
61e4c6ed	157	# https://github.com/ytdl-org/youtube-dl/pull/25216/files
	158	# The following is based on the pull request on the line above. Changed the ABC.com URL to a show available now.
	159	# https://abc.com/shows/the-rookie/episode-guide/season-02/19-the-q-word
	160	r'\bvideoIdCode["\']\s:\s["\'](vdka\w+)',
	161	# Deprecated fallback pattern
dd90a21c S	162	r'\b(?:video)?id["\']\s:\s["\'](VDKA\w+)'
dd90a21c S	163	), webpage, 'video id', default=video_id)
52007de8 S	164	if not site_info:
	165	brand = self._search_regex(
	166	(r'data-brand=\s["\']\s(\d+)',
	167	r'data-page-brand=\s["\']\s(\d+)'), webpage, 'brand',
	168	default='004')
	169	site_info = next(
	170	si for _, si in self._SITE_INFO.items()
	171	if si.get('brand') == brand)
bf2a5555 RA	172	if not video_id:
	173	# show extraction works for Disney, DisneyJunior and DisneyXD
	174	# ABC and Freeform has different layout
	175	show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
	176	videos = self._extract_videos(brand, show_id=show_id)
	177	show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
	178	entries = []
	179	for video in videos:
	180	entries.append(self.url_result(
	181	video['url'], 'Go', video.get('id'), video.get('title')))
	182	entries.reverse()
	183	return self.playlist_result(entries, show_id, show_title)
	184	video_data = self._extract_videos(brand, video_id)[0]
	185	video_id = video_data['id']
2c3e0af9 RA	186	title = video_data['title']
	187
	188	formats = []
	189	for asset in video_data.get('assets', {}).get('asset', []):
	190	asset_url = asset.get('value')
	191	if not asset_url:
	192	continue
	193	format_id = asset.get('format')
	194	ext = determine_ext(asset_url)
	195	if ext == 'm3u8':
014b7e6b	196	video_type = video_data.get('type')
8e1409fd RA	197	data = {
	198	'video_id': video_data['id'],
	199	'video_type': video_type,
	200	'brand': brand,
	201	'device': '001',
	202	}
	203	if video_data.get('accesslevel') == '1':
118afcf5 RA	204	requestor_id = site_info.get('requestor_id', 'DisneyChannels')
118afcf5 RA	205	resource = site_info.get('resource_id') or self._get_mvpd_resource(
8e1409fd RA	206	requestor_id, title, video_id, None)
	207	auth = self._extract_mvpd_auth(
	208	url, video_id, requestor_id, resource)
	209	data.update({
	210	'token': auth,
	211	'token_type': 'ap',
	212	'adobe_requestor_id': requestor_id,
	213	})
	214	else:
5f95927a	215	self._initialize_geo_bypass({'countries': ['US']})
8e1409fd RA	216	entitlement = self._download_json(
8e1409fd RA	217	'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
bf2a5555	218	video_id, data=urlencode_postdata(data))
8e1409fd RA	219	errors = entitlement.get('errors', {}).get('errors', [])
	220	if errors:
	221	for error in errors:
	222	if error.get('code') == 1002:
	223	self.raise_geo_restricted(
	224	error['message'], countries=['US'])
	225	error_message = ', '.join([error['message'] for error in errors])
	226	raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
	227	asset_url += '?' + entitlement['uplynkData']['sessionKey']
2c3e0af9 RA	228	formats.extend(self._extract_m3u8_formats(
	229	asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False))
	230	else:
8e1409fd	231	f = {
2c3e0af9 RA	232	'format_id': format_id,
	233	'url': asset_url,
	234	'ext': ext,
8e1409fd RA	235	}
	236	if re.search(r'(?:/mp4/source/\|_source\.mp4)', asset_url):
	237	f.update({
	238	'format_id': ('%s-' % format_id if format_id else '') + 'SOURCE',
f983b875	239	'quality': 1,
8e1409fd RA	240	})
	241	else:
	242	mobj = re.search(r'/(\d+)x(\d+)/', asset_url)
	243	if mobj:
	244	height = int(mobj.group(2))
	245	f.update({
	246	'format_id': ('%s-' % format_id if format_id else '') + '%dP' % height,
	247	'width': int(mobj.group(1)),
	248	'height': height,
	249	})
	250	formats.append(f)
2c3e0af9 RA	251	self._sort_formats(formats)
	252
	253	subtitles = {}
	254	for cc in video_data.get('closedcaption', {}).get('src', []):
	255	cc_url = cc.get('value')
	256	if not cc_url:
	257	continue
	258	ext = determine_ext(cc_url)
	259	if ext == 'xml':
	260	ext = 'ttml'
	261	subtitles.setdefault(cc.get('lang'), []).append({
	262	'url': cc_url,
	263	'ext': ext,
	264	})
	265
	266	thumbnails = []
	267	for thumbnail in video_data.get('thumbnails', {}).get('thumbnail', []):
	268	thumbnail_url = thumbnail.get('value')
	269	if not thumbnail_url:
	270	continue
	271	thumbnails.append({
	272	'url': thumbnail_url,
	273	'width': int_or_none(thumbnail.get('width')),
	274	'height': int_or_none(thumbnail.get('height')),
	275	})
	276
	277	return {
	278	'id': video_id,
	279	'title': title,
	280	'description': video_data.get('longdescription') or video_data.get('description'),
	281	'duration': int_or_none(video_data.get('duration', {}).get('value'), 1000),
	282	'age_limit': parse_age_limit(video_data.get('tvrating', {}).get('rating')),
	283	'episode_number': int_or_none(video_data.get('episodenumber')),
	284	'series': video_data.get('show', {}).get('title'),
	285	'season_number': int_or_none(video_data.get('season', {}).get('num')),
	286	'thumbnails': thumbnails,
	287	'formats': formats,
	288	'subtitles': subtitles,
	289	}