[yt-dlp.git] / youtube_dl / extractor / svt.py

# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    determine_ext,
    dict_get,
    int_or_none,
    try_get,
    urljoin,
    compat_str,
)


class SVTBaseIE(InfoExtractor):
    _GEO_COUNTRIES = ['SE']

    def _extract_video(self, video_info, video_id):
        is_live = dict_get(video_info, ('live', 'simulcast'), default=False)
        m3u8_protocol = 'm3u8' if is_live else 'm3u8_native'
        formats = []
        for vr in video_info['videoReferences']:
            player_type = vr.get('playerType') or vr.get('format')
            vurl = vr['url']
            ext = determine_ext(vurl)
            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    vurl, video_id,
                    ext='mp4', entry_protocol=m3u8_protocol,
                    m3u8_id=player_type, fatal=False))
            elif ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
                    vurl + '?hdcore=3.3.0', video_id,
                    f4m_id=player_type, fatal=False))
            elif ext == 'mpd':
                if player_type == 'dashhbbtv':
                    formats.extend(self._extract_mpd_formats(
                        vurl, video_id, mpd_id=player_type, fatal=False))
            else:
                formats.append({
                    'format_id': player_type,
                    'url': vurl,
                })
        if not formats and video_info.get('rights', {}).get('geoBlockedSweden'):
            self.raise_geo_restricted(
                'This video is only available in Sweden',
                countries=self._GEO_COUNTRIES)
        self._sort_formats(formats)

        subtitles = {}
        subtitle_references = dict_get(video_info, ('subtitles', 'subtitleReferences'))
        if isinstance(subtitle_references, list):
            for sr in subtitle_references:
                subtitle_url = sr.get('url')
                subtitle_lang = sr.get('language', 'sv')
                if subtitle_url:
                    if determine_ext(subtitle_url) == 'm3u8':
                        # TODO(yan12125): handle WebVTT in m3u8 manifests
                        continue

                    subtitles.setdefault(subtitle_lang, []).append({'url': subtitle_url})

        title = video_info.get('title')

        series = video_info.get('programTitle')
        season_number = int_or_none(video_info.get('season'))
        episode = video_info.get('episodeTitle')
        episode_number = int_or_none(video_info.get('episodeNumber'))

        duration = int_or_none(dict_get(video_info, ('materialLength', 'contentDuration')))
        age_limit = None
        adult = dict_get(
            video_info, ('inappropriateForChildren', 'blockedForChildren'),
            skip_false_values=False)
        if adult is not None:
            age_limit = 18 if adult else 0

        return {
            'id': video_id,
            'title': title,
            'formats': formats,
            'subtitles': subtitles,
            'duration': duration,
            'age_limit': age_limit,
            'series': series,
            'season_number': season_number,
            'episode': episode,
            'episode_number': episode_number,
            'is_live': is_live,
        }


class SVTIE(SVTBaseIE):
    _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P<widget_id>\d+)&.*?\barticleId=(?P<id>\d+)'
    _TEST = {
        'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
        'md5': '33e9a5d8f646523ce0868ecfb0eed77d',
        'info_dict': {
            'id': '2900353',
            'ext': 'mp4',
            'title': 'Stjärnorna skojar till det - under SVT-intervjun',
            'duration': 27,
            'age_limit': 0,
        },
    }

    @staticmethod
    def _extract_url(webpage):
        mobj = re.search(
            r'(?:<iframe src|href)="(?P<url>%s[^"]*)"' % SVTIE._VALID_URL, webpage)
        if mobj:
            return mobj.group('url')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        widget_id = mobj.group('widget_id')
        article_id = mobj.group('id')

        info = self._download_json(
            'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
            article_id)

        info_dict = self._extract_video(info['video'], article_id)
        info_dict['title'] = info['context']['title']
        return info_dict


class SVTPlayBaseIE(SVTBaseIE):
    _SVTPLAY_RE = r'root\s*\[\s*(["\'])_*svtplay\1\s*\]\s*=\s*(?P<json>{.+?})\s*;\s*\n'


class SVTPlayIE(SVTPlayBaseIE):
    IE_DESC = 'SVT Play and Öppet arkiv'
    _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
        'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
        'info_dict': {
            'id': '5996901',
            'ext': 'mp4',
            'title': 'Flygplan till Haile Selassie',
            'duration': 3527,
            'thumbnail': r're:^https?://.*[\.-]jpg$',
            'age_limit': 0,
            'subtitles': {
                'sv': [{
                    'ext': 'wsrt',
                }]
            },
        },
    }, {
        # geo restricted to Sweden
        'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
        'only_matching': True,
    }, {
        'url': 'http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg',
        'only_matching': True,
    }, {
        'url': 'https://www.svtplay.se/kanaler/svt1',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        data = self._parse_json(
            self._search_regex(
                self._SVTPLAY_RE, webpage, 'embedded data', default='{}',
                group='json'),
            video_id, fatal=False)

        thumbnail = self._og_search_thumbnail(webpage)

        def adjust_title(info):
            if info['is_live']:
                info['title'] = self._live_title(info['title'])

        if data:
            video_info = try_get(
                data, lambda x: x['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video'],
                dict)
            if video_info:
                info_dict = self._extract_video(video_info, video_id)
                info_dict.update({
                    'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
                    'thumbnail': thumbnail,
                })
                adjust_title(info_dict)
                return info_dict

        video_id = self._search_regex(
            r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
            webpage, 'video id', default=None)

        if video_id:
            data = self._download_json(
                'https://api.svt.se/videoplayer-api/video/%s' % video_id,
                video_id, headers=self.geo_verification_headers())
            info_dict = self._extract_video(data, video_id)
            if not info_dict.get('title'):
                info_dict['title'] = re.sub(
                    r'\s*\|\s*.+?$', '',
                    info_dict.get('episode') or self._og_search_title(webpage))
            adjust_title(info_dict)
            return info_dict


class SVTSeriesIE(SVTPlayBaseIE):
    _VALID_URL = r'https?://(?:www\.)?svtplay\.se/(?P<id>[^/?&#]+)'
    _TESTS = [{
        'url': 'https://www.svtplay.se/rederiet',
        'info_dict': {
            'id': 'rederiet',
            'title': 'Rederiet',
            'description': 'md5:505d491a58f4fcf6eb418ecab947e69e',
        },
        'playlist_mincount': 318,
    }, {
        'url': 'https://www.svtplay.se/rederiet?tab=sasong2',
        'info_dict': {
            'id': 'rederiet-sasong2',
            'title': 'Rederiet - Säsong 2',
            'description': 'md5:505d491a58f4fcf6eb418ecab947e69e',
        },
        'playlist_count': 12,
    }]

    @classmethod
    def suitable(cls, url):
        return False if SVTIE.suitable(url) or SVTPlayIE.suitable(url) else super(SVTSeriesIE, cls).suitable(url)

    def _real_extract(self, url):
        series_id = self._match_id(url)

        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
        season_slug = qs.get('tab', [None])[0]

        if season_slug:
            series_id += '-%s' % season_slug

        webpage = self._download_webpage(
            url, series_id, 'Downloading series page')

        root = self._parse_json(
            self._search_regex(
                self._SVTPLAY_RE, webpage, 'content', group='json'),
            series_id)

        season_name = None

        entries = []
        for season in root['relatedVideoContent']['relatedVideosAccordion']:
            if not isinstance(season, dict):
                continue
            if season_slug:
                if season.get('slug') != season_slug:
                    continue
                season_name = season.get('name')
            videos = season.get('videos')
            if not isinstance(videos, list):
                continue
            for video in videos:
                content_url = video.get('contentUrl')
                if not content_url or not isinstance(content_url, compat_str):
                    continue
                entries.append(
                    self.url_result(
                        urljoin(url, content_url),
                        ie=SVTPlayIE.ie_key(),
                        video_title=video.get('title')
                    ))

        metadata = root.get('metaData')
        if not isinstance(metadata, dict):
            metadata = {}

        title = metadata.get('title')
        season_name = season_name or season_slug

        if title and season_name:
            title = '%s - %s' % (title, season_name)
        elif season_slug:
            title = season_slug

        return self.playlist_result(
            entries, series_id, title, metadata.get('description'))
Commit	Line	Data
28f12728	1	# coding: utf-8
1309b396 PH	2	from __future__ import unicode_literals
1309b396 PH	3
df5ae3eb S	4	import re
df5ae3eb S	5
1309b396	6	from .common import InfoExtractor
df146eb2 S	7	from ..compat import (
	8	compat_parse_qs,
	9	compat_urllib_parse_urlparse,
	10	)
1309b396 PH	11	from ..utils import (
1309b396 PH	12	determine_ext,
e4f90ea0	13	dict_get,
23bdae09 S	14	int_or_none,
23bdae09 S	15	try_get,
fd97fa7b MW	16	urljoin,
fd97fa7b MW	17	compat_str,
1309b396 PH	18	)
	19
	20
79998cd5	21	class SVTBaseIE(InfoExtractor):
4248dad9	22	_GEO_COUNTRIES = ['SE']
6d4c2597	23
23bdae09	24	def _extract_video(self, video_info, video_id):
488ff2dd	25	is_live = dict_get(video_info, ('live', 'simulcast'), default=False)
488ff2dd	26	m3u8_protocol = 'm3u8' if is_live else 'm3u8_native'
1309b396 PH	27	formats = []
1309b396 PH	28	for vr in video_info['videoReferences']:
21d21b0c	29	player_type = vr.get('playerType') or vr.get('format')
1309b396	30	vurl = vr['url']
df5ae3eb S	31	ext = determine_ext(vurl)
df5ae3eb S	32	if ext == 'm3u8':
1309b396 PH	33	formats.extend(self._extract_m3u8_formats(
1309b396 PH	34	vurl, video_id,
488ff2dd	35	ext='mp4', entry_protocol=m3u8_protocol,
edfd9351	36	m3u8_id=player_type, fatal=False))
df5ae3eb S	37	elif ext == 'f4m':
	38	formats.extend(self._extract_f4m_formats(
	39	vurl + '?hdcore=3.3.0', video_id,
edfd9351	40	f4m_id=player_type, fatal=False))
	41	elif ext == 'mpd':
	42	if player_type == 'dashhbbtv':
	43	formats.extend(self._extract_mpd_formats(
	44	vurl, video_id, mpd_id=player_type, fatal=False))
1309b396 PH	45	else:
1309b396 PH	46	formats.append({
edfd9351	47	'format_id': player_type,
1309b396 PH	48	'url': vurl,
1309b396 PH	49	})
23bdae09	50	if not formats and video_info.get('rights', {}).get('geoBlockedSweden'):
04d906ea	51	self.raise_geo_restricted(
4248dad9 S	52	'This video is only available in Sweden',
4248dad9 S	53	countries=self._GEO_COUNTRIES)
1309b396 PH	54	self._sort_formats(formats)
1309b396 PH	55
1f16b958	56	subtitles = {}
e4f90ea0	57	subtitle_references = dict_get(video_info, ('subtitles', 'subtitleReferences'))
594c4d79 S	58	if isinstance(subtitle_references, list):
	59	for sr in subtitle_references:
	60	subtitle_url = sr.get('url')
e4f90ea0	61	subtitle_lang = sr.get('language', 'sv')
594c4d79	62	if subtitle_url:
e4f90ea0 YCH	63	if determine_ext(subtitle_url) == 'm3u8':
	64	# TODO(yan12125): handle WebVTT in m3u8 manifests
	65	continue
	66
	67	subtitles.setdefault(subtitle_lang, []).append({'url': subtitle_url})
1f16b958	68
23bdae09 S	69	title = video_info.get('title')
	70
	71	series = video_info.get('programTitle')
	72	season_number = int_or_none(video_info.get('season'))
	73	episode = video_info.get('episodeTitle')
	74	episode_number = int_or_none(video_info.get('episodeNumber'))
	75
	76	duration = int_or_none(dict_get(video_info, ('materialLength', 'contentDuration')))
	77	age_limit = None
	78	adult = dict_get(
	79	video_info, ('inappropriateForChildren', 'blockedForChildren'),
	80	skip_false_values=False)
	81	if adult is not None:
	82	age_limit = 18 if adult else 0
1309b396 PH	83
	84	return {
	85	'id': video_id,
23bdae09	86	'title': title,
1309b396	87	'formats': formats,
1f16b958	88	'subtitles': subtitles,
1309b396	89	'duration': duration,
df5ae3eb	90	'age_limit': age_limit,
23bdae09 S	91	'series': series,
	92	'season_number': season_number,
	93	'episode': episode,
	94	'episode_number': episode_number,
488ff2dd	95	'is_live': is_live,
1309b396	96	}
79998cd5 S	97
	98
	99	class SVTIE(SVTBaseIE):
	100	_VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.?&)?widgetId=(?P<widget_id>\d+)&.?\barticleId=(?P<id>\d+)'
	101	_TEST = {
	102	'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
e4f90ea0	103	'md5': '33e9a5d8f646523ce0868ecfb0eed77d',
79998cd5 S	104	'info_dict': {
79998cd5 S	105	'id': '2900353',
e4f90ea0 YCH	106	'ext': 'mp4',
e4f90ea0 YCH	107	'title': 'Stjärnorna skojar till det - under SVT-intervjun',
79998cd5 S	108	'duration': 27,
	109	'age_limit': 0,
	110	},
	111	}
	112
bab19a8e S	113	@staticmethod
	114	def _extract_url(webpage):
	115	mobj = re.search(
	116	r'(?:<iframe src\|href)="(?P<url>%s[^"]*)"' % SVTIE._VALID_URL, webpage)
	117	if mobj:
	118	return mobj.group('url')
	119
79998cd5 S	120	def _real_extract(self, url):
	121	mobj = re.match(self._VALID_URL, url)
	122	widget_id = mobj.group('widget_id')
	123	article_id = mobj.group('id')
e4f90ea0 YCH	124
e4f90ea0 YCH	125	info = self._download_json(
79998cd5 S	126	'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
	127	article_id)
	128
23bdae09	129	info_dict = self._extract_video(info['video'], article_id)
e4f90ea0 YCH	130	info_dict['title'] = info['context']['title']
	131	return info_dict
	132
79998cd5	133
1236ac6b S	134	class SVTPlayBaseIE(SVTBaseIE):
	135	_SVTPLAY_RE = r'root\s\[\s(["\'])_svtplay\1\s\]\s=\s(?P<json>{.+?})\s;\s\n'
	136
	137
	138	class SVTPlayIE(SVTPlayBaseIE):
79998cd5	139	IE_DESC = 'SVT Play and Öppet arkiv'
6cdaaf70	140	_VALID_URL = r'https?://(?:www\.)?(?:svtplay\|oppetarkiv)\.se/(?:video\|klipp\|kanaler)/(?P<id>[^/?#&]+)'
23bdae09	141	_TESTS = [{
594c4d79 S	142	'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
594c4d79 S	143	'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
79998cd5	144	'info_dict': {
594c4d79 S	145	'id': '5996901',
	146	'ext': 'mp4',
	147	'title': 'Flygplan till Haile Selassie',
	148	'duration': 3527,
ec85ded8	149	'thumbnail': r're:^https?://.*[\.-]jpg$',
79998cd5	150	'age_limit': 0,
594c4d79 S	151	'subtitles': {
	152	'sv': [{
	153	'ext': 'wsrt',
	154	}]
	155	},
79998cd5	156	},
23bdae09 S	157	}, {
	158	# geo restricted to Sweden
	159	'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
	160	'only_matching': True,
3b34ab53 S	161	}, {
	162	'url': 'http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg',
	163	'only_matching': True,
488ff2dd	164	}, {
	165	'url': 'https://www.svtplay.se/kanaler/svt1',
	166	'only_matching': True,
23bdae09	167	}]
e4f90ea0	168
79998cd5	169	def _real_extract(self, url):
e4f90ea0 YCH	170	video_id = self._match_id(url)
	171
	172	webpage = self._download_webpage(url, video_id)
	173
23bdae09 S	174	data = self._parse_json(
23bdae09 S	175	self._search_regex(
1236ac6b S	176	self._SVTPLAY_RE, webpage, 'embedded data', default='{}',
1236ac6b S	177	group='json'),
23bdae09	178	video_id, fatal=False)
e4f90ea0 YCH	179
	180	thumbnail = self._og_search_thumbnail(webpage)
	181
6cdaaf70 S	182	def adjust_title(info):
	183	if info['is_live']:
	184	info['title'] = self._live_title(info['title'])
	185
23bdae09 S	186	if data:
	187	video_info = try_get(
	188	data, lambda x: x['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video'],
	189	dict)
	190	if video_info:
	191	info_dict = self._extract_video(video_info, video_id)
	192	info_dict.update({
	193	'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
	194	'thumbnail': thumbnail,
	195	})
6cdaaf70	196	adjust_title(info_dict)
23bdae09 S	197	return info_dict
	198
	199	video_id = self._search_regex(
	200	r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
	201	webpage, 'video id', default=None)
	202
	203	if video_id:
	204	data = self._download_json(
c0401751 S	205	'https://api.svt.se/videoplayer-api/video/%s' % video_id,
c0401751 S	206	video_id, headers=self.geo_verification_headers())
23bdae09 S	207	info_dict = self._extract_video(data, video_id)
	208	if not info_dict.get('title'):
	209	info_dict['title'] = re.sub(
	210	r'\s\\|\s.+?$', '',
	211	info_dict.get('episode') or self._og_search_title(webpage))
6cdaaf70	212	adjust_title(info_dict)
23bdae09	213	return info_dict
fd97fa7b MW	214
fd97fa7b MW	215
1236ac6b	216	class SVTSeriesIE(SVTPlayBaseIE):
fd97fa7b	217	_VALID_URL = r'https?://(?:www\.)?svtplay\.se/(?P<id>[^/?&#]+)'
fd97fa7b MW	218	_TESTS = [{
	219	'url': 'https://www.svtplay.se/rederiet',
	220	'info_dict': {
	221	'id': 'rederiet',
	222	'title': 'Rederiet',
	223	'description': 'md5:505d491a58f4fcf6eb418ecab947e69e',
	224	},
	225	'playlist_mincount': 318,
df146eb2 S	226	}, {
	227	'url': 'https://www.svtplay.se/rederiet?tab=sasong2',
	228	'info_dict': {
	229	'id': 'rederiet-sasong2',
	230	'title': 'Rederiet - Säsong 2',
	231	'description': 'md5:505d491a58f4fcf6eb418ecab947e69e',
	232	},
	233	'playlist_count': 12,
fd97fa7b MW	234	}]
	235
	236	@classmethod
	237	def suitable(cls, url):
b71bb3ba	238	return False if SVTIE.suitable(url) or SVTPlayIE.suitable(url) else super(SVTSeriesIE, cls).suitable(url)
fd97fa7b MW	239
fd97fa7b MW	240	def _real_extract(self, url):
df146eb2 S	241	series_id = self._match_id(url)
	242
	243	qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
	244	season_slug = qs.get('tab', [None])[0]
	245
	246	if season_slug:
	247	series_id += '-%s' % season_slug
fd97fa7b	248
b71bb3ba	249	webpage = self._download_webpage(
df146eb2	250	url, series_id, 'Downloading series page')
fd97fa7b	251
b71bb3ba S	252	root = self._parse_json(
b71bb3ba S	253	self._search_regex(
1236ac6b	254	self._SVTPLAY_RE, webpage, 'content', group='json'),
df146eb2 S	255	series_id)
	256
	257	season_name = None
fd97fa7b MW	258
fd97fa7b MW	259	entries = []
b71bb3ba	260	for season in root['relatedVideoContent']['relatedVideosAccordion']:
df146eb2 S	261	if not isinstance(season, dict):
	262	continue
	263	if season_slug:
	264	if season.get('slug') != season_slug:
	265	continue
	266	season_name = season.get('name')
fd97fa7b MW	267	videos = season.get('videos')
	268	if not isinstance(videos, list):
	269	continue
fd97fa7b MW	270	for video in videos:
fd97fa7b MW	271	content_url = video.get('contentUrl')
b71bb3ba	272	if not content_url or not isinstance(content_url, compat_str):
fd97fa7b MW	273	continue
	274	entries.append(
	275	self.url_result(
	276	urljoin(url, content_url),
	277	ie=SVTPlayIE.ie_key(),
	278	video_title=video.get('title')
	279	))
	280
b71bb3ba S	281	metadata = root.get('metaData')
	282	if not isinstance(metadata, dict):
	283	metadata = {}
	284
df146eb2 S	285	title = metadata.get('title')
	286	season_name = season_name or season_slug
	287
	288	if title and season_name:
	289	title = '%s - %s' % (title, season_name)
	290	elif season_slug:
	291	title = season_slug
	292
fd97fa7b	293	return self.playlist_result(
df146eb2	294	entries, series_id, title, metadata.get('description'))