[yt-dlp.git] / yt_dlp / extractor / sky.py

import re

from .common import InfoExtractor
from ..utils import (
    extract_attributes,
    smuggle_url,
    strip_or_none,
    urljoin,
)


class SkyBaseIE(InfoExtractor):
    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
    _SDC_EL_REGEX = r'(?s)(<div[^>]+data-(?:component-name|fn)="sdc-(?:articl|sit)e-video"[^>]*>)'

    def _process_ooyala_element(self, webpage, sdc_el, url):
        sdc = extract_attributes(sdc_el)
        provider = sdc.get('data-provider')
        if provider == 'ooyala':
            video_id = sdc['data-sdc-video-id']
            video_url = 'ooyala:%s' % video_id
            ie_key = 'Ooyala'
            ooyala_el = self._search_regex(
                r'(<div[^>]+class="[^"]*\bsdc-article-video__media-ooyala\b[^"]*"[^>]+data-video-id="%s"[^>]*>)' % video_id,
                webpage, 'video data', fatal=False)
            if ooyala_el:
                ooyala_attrs = extract_attributes(ooyala_el) or {}
                if ooyala_attrs.get('data-token-required') == 'true':
                    token_fetch_url = (self._parse_json(ooyala_attrs.get(
                        'data-token-fetch-options', '{}'),
                        video_id, fatal=False) or {}).get('url')
                    if token_fetch_url:
                        embed_token = self._download_json(urljoin(
                            url, token_fetch_url), video_id, fatal=False)
                        if embed_token:
                            video_url = smuggle_url(
                                video_url, {'embed_token': embed_token})
        elif provider == 'brightcove':
            video_id = sdc['data-video-id']
            account_id = sdc.get('data-account-id') or '6058004172001'
            player_id = sdc.get('data-player-id') or 'RC9PQUaJ6'
            video_url = self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id)
            ie_key = 'BrightcoveNew'

        return {
            '_type': 'url_transparent',
            'id': video_id,
            'url': video_url,
            'ie_key': ie_key,
        }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        info = self._process_ooyala_element(webpage, self._search_regex(
            self._SDC_EL_REGEX, webpage, 'sdc element'), url)
        info.update({
            'title': self._og_search_title(webpage),
            'description': strip_or_none(self._og_search_description(webpage)),
        })
        return info


class SkySportsIE(SkyBaseIE):
    IE_NAME = 'sky:sports'
    _VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/([^/]+/)*(?P<id>[0-9]+)'
    _TESTS = [{
        'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
        'md5': '77d59166cddc8d3cb7b13e35eaf0f5ec',
        'info_dict': {
            'id': 'o3eWJnNDE6l7kfNO8BOoBlRxXRQ4ANNQ',
            'ext': 'mp4',
            'title': 'Bale: It\'s our time to shine',
            'description': 'md5:e88bda94ae15f7720c5cb467e777bb6d',
        },
        'add_ie': ['Ooyala'],
    }, {
        'url': 'https://www.skysports.com/watch/video/sports/f1/12160544/abu-dhabi-gp-the-notebook',
        'only_matching': True,
    }, {
        'url': 'https://www.skysports.com/watch/video/tv-shows/12118508/rainford-brent-how-ace-programme-helps',
        'only_matching': True,
    }]


class SkyNewsIE(SkyBaseIE):
    IE_NAME = 'sky:news'
    _VALID_URL = r'https?://news\.sky\.com/video/[0-9a-z-]+-(?P<id>[0-9]+)'
    _TEST = {
        'url': 'https://news.sky.com/video/russian-plane-inspected-after-deadly-fire-11712962',
        'md5': '411e8893fd216c75eaf7e4c65d364115',
        'info_dict': {
            'id': 'ref:1ua21xaDE6lCtZDmbYfl8kwsKLooJbNM',
            'ext': 'mp4',
            'title': 'Russian plane inspected after deadly fire',
            'description': 'The Russian Investigative Committee has released video of the wreckage of a passenger plane which caught fire near Moscow.',
            'uploader_id': '6058004172001',
            'timestamp': 1567112345,
            'upload_date': '20190829',
        },
        'add_ie': ['BrightcoveNew'],
    }


class SkyNewsStoryIE(SkyBaseIE):
    IE_NAME = 'sky:news:story'
    _VALID_URL = r'https?://news\.sky\.com/story/[0-9a-z-]+-(?P<id>[0-9]+)'
    _TEST = {
        'url': 'https://news.sky.com/story/budget-2021-chancellor-rishi-sunak-vows-address-will-deliver-strong-economy-fit-for-a-new-age-of-optimism-12445425',
        'info_dict': {
            'id': 'ref:0714acb9-123d-42c8-91b8-5c1bc6c73f20',
            'title': 'md5:e408dd7aad63f31a1817bbe40c7d276f',
            'description': 'md5:a881e12f49212f92be2befe4a09d288a',
            'ext': 'mp4',
            'upload_date': '20211027',
            'timestamp': 1635317494,
            'uploader_id': '6058004172001',
        }
    }

    def _real_extract(self, url):
        article_id = self._match_id(url)
        webpage = self._download_webpage(url, article_id)

        entries = [self._process_ooyala_element(webpage, sdc_el, url)
                   for sdc_el in re.findall(self._SDC_EL_REGEX, webpage)]

        return self.playlist_result(
            entries, article_id, self._og_search_title(webpage),
            self._html_search_meta(['og:description', 'description'], webpage))


class SkySportsNewsIE(SkyBaseIE):
    IE_NAME = 'sky:sports:news'
    _VALID_URL = r'https?://(?:www\.)?skysports\.com/([^/]+/)*news/\d+/(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.skysports.com/golf/news/12176/10871916/dustin-johnson-ready-to-conquer-players-championship-at-tpc-sawgrass',
        'info_dict': {
            'id': '10871916',
            'title': 'Dustin Johnson ready to conquer Players Championship at TPC Sawgrass',
            'description': 'Dustin Johnson is confident he can continue his dominant form in 2017 by adding the Players Championship to his list of victories.',
        },
        'playlist_count': 2,
    }

    def _real_extract(self, url):
        article_id = self._match_id(url)
        webpage = self._download_webpage(url, article_id)

        entries = []
        for sdc_el in re.findall(self._SDC_EL_REGEX, webpage):
            entries.append(self._process_ooyala_element(webpage, sdc_el, url))

        return self.playlist_result(
            entries, article_id, self._og_search_title(webpage),
            self._html_search_meta(['og:description', 'description'], webpage))
Commit	Line	Data
29f7c58a	1	import re
29f7c58a	2
df43389a	3	from .common import InfoExtractor
44cca168 RA	4	from ..utils import (
	5	extract_attributes,
	6	smuggle_url,
	7	strip_or_none,
	8	urljoin,
	9	)
df43389a RA	10
df43389a RA	11
71ebd35d	12	class SkyBaseIE(InfoExtractor):
29f7c58a	13	BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
	14	_SDC_EL_REGEX = r'(?s)(<div[^>]+data-(?:component-name\|fn)="sdc-(?:articl\|sit)e-video"[^>]*>)'
	15
	16	def _process_ooyala_element(self, webpage, sdc_el, url):
	17	sdc = extract_attributes(sdc_el)
	18	provider = sdc.get('data-provider')
	19	if provider == 'ooyala':
	20	video_id = sdc['data-sdc-video-id']
	21	video_url = 'ooyala:%s' % video_id
	22	ie_key = 'Ooyala'
	23	ooyala_el = self._search_regex(
	24	r'(<div[^>]+class="[^"]\bsdc-article-video__media-ooyala\b[^"]"[^>]+data-video-id="%s"[^>]*>)' % video_id,
	25	webpage, 'video data', fatal=False)
	26	if ooyala_el:
	27	ooyala_attrs = extract_attributes(ooyala_el) or {}
	28	if ooyala_attrs.get('data-token-required') == 'true':
	29	token_fetch_url = (self._parse_json(ooyala_attrs.get(
	30	'data-token-fetch-options', '{}'),
	31	video_id, fatal=False) or {}).get('url')
	32	if token_fetch_url:
	33	embed_token = self._download_json(urljoin(
	34	url, token_fetch_url), video_id, fatal=False)
	35	if embed_token:
	36	video_url = smuggle_url(
	37	video_url, {'embed_token': embed_token})
	38	elif provider == 'brightcove':
	39	video_id = sdc['data-video-id']
	40	account_id = sdc.get('data-account-id') or '6058004172001'
	41	player_id = sdc.get('data-player-id') or 'RC9PQUaJ6'
	42	video_url = self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id)
	43	ie_key = 'BrightcoveNew'
df43389a RA	44
	45	return {
	46	'_type': 'url_transparent',
	47	'id': video_id,
44cca168	48	'url': video_url,
29f7c58a	49	'ie_key': ie_key,
	50	}
	51
	52	def _real_extract(self, url):
	53	video_id = self._match_id(url)
	54	webpage = self._download_webpage(url, video_id)
	55	info = self._process_ooyala_element(webpage, self._search_regex(
	56	self._SDC_EL_REGEX, webpage, 'sdc element'), url)
	57	info.update({
df43389a	58	'title': self._og_search_title(webpage),
a814da3f	59	'description': strip_or_none(self._og_search_description(webpage)),
29f7c58a	60	})
29f7c58a	61	return info
71ebd35d RA	62
	63
	64	class SkySportsIE(SkyBaseIE):
29f7c58a	65	IE_NAME = 'sky:sports'
	66	_VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/([^/]+/)*(?P<id>[0-9]+)'
	67	_TESTS = [{
71ebd35d RA	68	'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
	69	'md5': '77d59166cddc8d3cb7b13e35eaf0f5ec',
	70	'info_dict': {
	71	'id': 'o3eWJnNDE6l7kfNO8BOoBlRxXRQ4ANNQ',
	72	'ext': 'mp4',
	73	'title': 'Bale: It\'s our time to shine',
	74	'description': 'md5:e88bda94ae15f7720c5cb467e777bb6d',
	75	},
	76	'add_ie': ['Ooyala'],
29f7c58a	77	}, {
	78	'url': 'https://www.skysports.com/watch/video/sports/f1/12160544/abu-dhabi-gp-the-notebook',
	79	'only_matching': True,
	80	}, {
	81	'url': 'https://www.skysports.com/watch/video/tv-shows/12118508/rainford-brent-how-ace-programme-helps',
	82	'only_matching': True,
	83	}]
71ebd35d RA	84
	85
	86	class SkyNewsIE(SkyBaseIE):
29f7c58a	87	IE_NAME = 'sky:news'
71ebd35d RA	88	_VALID_URL = r'https?://news\.sky\.com/video/[0-9a-z-]+-(?P<id>[0-9]+)'
	89	_TEST = {
	90	'url': 'https://news.sky.com/video/russian-plane-inspected-after-deadly-fire-11712962',
29f7c58a	91	'md5': '411e8893fd216c75eaf7e4c65d364115',
71ebd35d	92	'info_dict': {
29f7c58a	93	'id': 'ref:1ua21xaDE6lCtZDmbYfl8kwsKLooJbNM',
71ebd35d RA	94	'ext': 'mp4',
	95	'title': 'Russian plane inspected after deadly fire',
	96	'description': 'The Russian Investigative Committee has released video of the wreckage of a passenger plane which caught fire near Moscow.',
29f7c58a	97	'uploader_id': '6058004172001',
	98	'timestamp': 1567112345,
	99	'upload_date': '20190829',
71ebd35d	100	},
29f7c58a	101	'add_ie': ['BrightcoveNew'],
	102	}
	103
	104
b7b186e7	105	class SkyNewsStoryIE(SkyBaseIE):
	106	IE_NAME = 'sky:news:story'
	107	_VALID_URL = r'https?://news\.sky\.com/story/[0-9a-z-]+-(?P<id>[0-9]+)'
	108	_TEST = {
	109	'url': 'https://news.sky.com/story/budget-2021-chancellor-rishi-sunak-vows-address-will-deliver-strong-economy-fit-for-a-new-age-of-optimism-12445425',
	110	'info_dict': {
	111	'id': 'ref:0714acb9-123d-42c8-91b8-5c1bc6c73f20',
	112	'title': 'md5:e408dd7aad63f31a1817bbe40c7d276f',
	113	'description': 'md5:a881e12f49212f92be2befe4a09d288a',
	114	'ext': 'mp4',
	115	'upload_date': '20211027',
	116	'timestamp': 1635317494,
	117	'uploader_id': '6058004172001',
	118	}
	119	}
	120
	121	def _real_extract(self, url):
	122	article_id = self._match_id(url)
	123	webpage = self._download_webpage(url, article_id)
	124
	125	entries = [self._process_ooyala_element(webpage, sdc_el, url)
	126	for sdc_el in re.findall(self._SDC_EL_REGEX, webpage)]
	127
	128	return self.playlist_result(
	129	entries, article_id, self._og_search_title(webpage),
	130	self._html_search_meta(['og:description', 'description'], webpage))
	131
	132
29f7c58a	133	class SkySportsNewsIE(SkyBaseIE):
	134	IE_NAME = 'sky:sports:news'
	135	_VALID_URL = r'https?://(?:www\.)?skysports\.com/([^/]+/)*news/\d+/(?P<id>\d+)'
	136	_TEST = {
	137	'url': 'http://www.skysports.com/golf/news/12176/10871916/dustin-johnson-ready-to-conquer-players-championship-at-tpc-sawgrass',
	138	'info_dict': {
	139	'id': '10871916',
	140	'title': 'Dustin Johnson ready to conquer Players Championship at TPC Sawgrass',
	141	'description': 'Dustin Johnson is confident he can continue his dominant form in 2017 by adding the Players Championship to his list of victories.',
	142	},
	143	'playlist_count': 2,
71ebd35d	144	}
29f7c58a	145
	146	def _real_extract(self, url):
	147	article_id = self._match_id(url)
	148	webpage = self._download_webpage(url, article_id)
	149
	150	entries = []
	151	for sdc_el in re.findall(self._SDC_EL_REGEX, webpage):
	152	entries.append(self._process_ooyala_element(webpage, sdc_el, url))
	153
	154	return self.playlist_result(
	155	entries, article_id, self._og_search_title(webpage),
	156	self._html_search_meta(['og:description', 'description'], webpage))