[yt-dlp.git] / youtube_dl / extractor / drtv.py

# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    int_or_none,
    float_or_none,
    mimetype2ext,
    parse_iso8601,
    remove_end,
    update_url_query,
)


class DRTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv/se|nyheder|radio/ondemand)/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
    _GEO_BYPASS = False
    _GEO_COUNTRIES = ['DK']
    IE_NAME = 'drtv'
    _TESTS = [{
        'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
        'md5': '25e659cccc9a2ed956110a299fdf5983',
        'info_dict': {
            'id': 'klassen-darlig-taber-10',
            'ext': 'mp4',
            'title': 'Klassen - Dårlig taber (10)',
            'description': 'md5:815fe1b7fa656ed80580f31e8b3c79aa',
            'timestamp': 1471991907,
            'upload_date': '20160823',
            'duration': 606.84,
        },
        'params': {
            'skip_download': True,
        },
    }, {
        'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
        'md5': '2c37175c718155930f939ef59952474a',
        'info_dict': {
            'id': 'christiania-pusher-street-ryddes-drdkrjpo',
            'ext': 'mp4',
            'title': 'LIVE Christianias rydning af Pusher Street er i gang',
            'description': '- Det er det fedeste, der er sket i 20 år, fortæller christianit til DR Nyheder.',
            'timestamp': 1472800279,
            'upload_date': '20160902',
            'duration': 131.4,
        },
    }, {
        'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
        'md5': '7c8ca12e6c3d3e3edd59ba5a9b7ca10a',
        'info_dict': {
            'id': 'historien-om-danmark-stenalder',
            'ext': 'mp4',
            'title': 'Historien om Danmark: Stenalder (1)',
            'description': 'Én fascinerende historie om tusindvis af år, hvor vores land bliver skabt ud af is og vand, og hvor de første danskere ankommer til vores egn. Det bliver en rejse ind i urtiden og det liv, som urtidsjægerne har levet i skovene og ved havet og helt frem til bondestenalderen. Gennem skeletfund afslører eksperter, hvordan vores forfædre har set ud i stenalderen og hvorfor stenaldermennesket byggede de imponerende jættestuer, som ligger overalt i det danske.',
            'timestamp': 1490401996,
            'upload_date': '20170325',
            'duration': 3502.04,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        if '>Programmet er ikke længere tilgængeligt' in webpage:
            raise ExtractorError(
                'Video %s is not available' % video_id, expected=True)

        video_id = self._search_regex(
            (r'data-(?:material-identifier|episode-slug)="([^"]+)"',
                r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
            webpage, 'video id')

        programcard = self._download_json(
            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id,
            video_id, 'Downloading video JSON')
        data = programcard['Data'][0]

        title = remove_end(self._og_search_title(
            webpage, default=None), ' | TV | DR') or data['Title']
        description = self._og_search_description(
            webpage, default=None) or data.get('Description')

        timestamp = parse_iso8601(data.get('CreatedTime'))

        thumbnail = None
        duration = None

        restricted_to_denmark = False

        formats = []
        subtitles = {}

        for asset in data['Assets']:
            kind = asset.get('Kind')
            if kind == 'Image':
                thumbnail = asset.get('Uri')
            preference = 0

            sign_language = asset.get('Target') == 'SignLanguage'

            if kind in ('VideoResource', 'AudioResource'):
                duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
                restricted_to_denmark = asset.get('RestrictedToDenmark')
                spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
                for link in asset.get('Links', []):
                    uri = link.get('Uri')
                    if not uri:
                        continue
                    target = link.get('Target')
                    format_id = target or ''
                    if spoken_subtitles:
                        preference = -1
                        format_id += '-spoken-subtitles'
                    if sign_language:
                        preference = -1
                        format_id += "-sign-language"
                    if target == 'HDS':
                        f4m_formats = self._extract_f4m_formats(
                            uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
                            video_id, preference, f4m_id=format_id)
                        if kind == 'AudioResource':
                            for f in f4m_formats:
                                f['vcodec'] = 'none'
                        formats.extend(f4m_formats)
                    elif target == 'HLS':
                        formats.extend(self._extract_m3u8_formats(
                            uri, video_id, 'mp4', entry_protocol='m3u8_native',
                            preference=preference, m3u8_id=format_id))
                    else:
                        bitrate = link.get('Bitrate')
                        if bitrate:
                            format_id += '-%s' % bitrate
                        formats.append({
                            'url': uri,
                            'format_id': format_id,
                            'tbr': int_or_none(bitrate),
                            'ext': link.get('FileFormat'),
                            'vcodec': 'none' if kind == 'AudioResource' else None,
                        })
                subtitles_list = asset.get('SubtitlesList')
                if isinstance(subtitles_list, list):
                    LANGS = {
                        'Danish': 'da',
                    }
                    for subs in subtitles_list:
                        if not subs.get('Uri'):
                            continue
                        lang = subs.get('Language') or 'da'
                        subtitles.setdefault(LANGS.get(lang, lang), []).append({
                            'url': subs['Uri'],
                            'ext': mimetype2ext(subs.get('MimeType')) or 'vtt'
                        })

        if not formats and restricted_to_denmark:
            self.raise_geo_restricted(
                'Unfortunately, DR is not allowed to show this program outside Denmark.',
                countries=self._GEO_COUNTRIES)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': title,
            'description': description,
            'thumbnail': thumbnail,
            'timestamp': timestamp,
            'duration': duration,
            'formats': formats,
            'subtitles': subtitles,
        }


class DRTVLiveIE(InfoExtractor):
    IE_NAME = 'drtv:live'
    _VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv|TV)/live/(?P<id>[\da-z-]+)'
    _GEO_COUNTRIES = ['DK']
    _TEST = {
        'url': 'https://www.dr.dk/tv/live/dr1',
        'info_dict': {
            'id': 'dr1',
            'ext': 'mp4',
            'title': 're:^DR1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
        },
        'params': {
            # m3u8 download
            'skip_download': True,
        },
    }

    def _real_extract(self, url):
        channel_id = self._match_id(url)
        channel_data = self._download_json(
            'https://www.dr.dk/mu-online/api/1.0/channel/' + channel_id,
            channel_id)
        title = self._live_title(channel_data['Title'])

        formats = []
        for streaming_server in channel_data.get('StreamingServers', []):
            server = streaming_server.get('Server')
            if not server:
                continue
            link_type = streaming_server.get('LinkType')
            for quality in streaming_server.get('Qualities', []):
                for stream in quality.get('Streams', []):
                    stream_path = stream.get('Stream')
                    if not stream_path:
                        continue
                    stream_url = update_url_query(
                        '%s/%s' % (server, stream_path), {'b': ''})
                    if link_type == 'HLS':
                        formats.extend(self._extract_m3u8_formats(
                            stream_url, channel_id, 'mp4',
                            m3u8_id=link_type, fatal=False, live=True))
                    elif link_type == 'HDS':
                        formats.extend(self._extract_f4m_formats(update_url_query(
                            '%s/%s' % (server, stream_path), {'hdcore': '3.7.0'}),
                            channel_id, f4m_id=link_type, fatal=False))
        self._sort_formats(formats)

        return {
            'id': channel_id,
            'title': title,
            'thumbnail': channel_data.get('PrimaryImageUri'),
            'formats': formats,
            'is_live': True,
        }
Commit	Line	Data
7d254639	1	# coding: utf-8
f2b8db57 S	2	from __future__ import unicode_literals
f2b8db57 S	3
1335c3ac S	4	from .common import InfoExtractor
	5	from ..utils import (
	6	ExtractorError,
6066d03d S	7	int_or_none,
	8	float_or_none,
	9	mimetype2ext,
1335c3ac	10	parse_iso8601,
5e9e3d0f	11	remove_end,
2c15db82	12	update_url_query,
1335c3ac	13	)
f2b8db57 S	14
f2b8db57 S	15
18c1c424	16	class DRTVIE(InfoExtractor):
2c15db82	17	_VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv/se\|nyheder\|radio/ondemand)/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]\|$)'
96182695 S	18	_GEO_BYPASS = False
96182695 S	19	_GEO_COUNTRIES = ['DK']
2c15db82	20	IE_NAME = 'drtv'
5e9e3d0f	21	_TESTS = [{
3fcce302 S	22	'url': 'https://www.dr.dk/tv/se/boern/ultra/klassen-ultra/klassen-darlig-taber-10',
3fcce302 S	23	'md5': '25e659cccc9a2ed956110a299fdf5983',
f2b8db57	24	'info_dict': {
3fcce302	25	'id': 'klassen-darlig-taber-10',
f2b8db57	26	'ext': 'mp4',
3fcce302 S	27	'title': 'Klassen - Dårlig taber (10)',
	28	'description': 'md5:815fe1b7fa656ed80580f31e8b3c79aa',
	29	'timestamp': 1471991907,
	30	'upload_date': '20160823',
	31	'duration': 606.84,
	32	},
	33	'params': {
	34	'skip_download': True,
f2b8db57	35	},
5e9e3d0f SB	36	}, {
5e9e3d0f SB	37	'url': 'https://www.dr.dk/nyheder/indland/live-christianias-rydning-af-pusher-street-er-i-gang',
3fcce302	38	'md5': '2c37175c718155930f939ef59952474a',
5e9e3d0f SB	39	'info_dict': {
	40	'id': 'christiania-pusher-street-ryddes-drdkrjpo',
	41	'ext': 'mp4',
	42	'title': 'LIVE Christianias rydning af Pusher Street er i gang',
	43	'description': '- Det er det fedeste, der er sket i 20 år, fortæller christianit til DR Nyheder.',
	44	'timestamp': 1472800279,
	45	'upload_date': '20160902',
	46	'duration': 131.4,
3fcce302	47	},
b972fb03 RR	48	}, {
	49	'url': 'https://www.dr.dk/tv/se/historien-om-danmark/-/historien-om-danmark-stenalder',
	50	'md5': '7c8ca12e6c3d3e3edd59ba5a9b7ca10a',
	51	'info_dict': {
	52	'id': 'historien-om-danmark-stenalder',
	53	'ext': 'mp4',
	54	'title': 'Historien om Danmark: Stenalder (1)',
	55	'description': 'Én fascinerende historie om tusindvis af år, hvor vores land bliver skabt ud af is og vand, og hvor de første danskere ankommer til vores egn. Det bliver en rejse ind i urtiden og det liv, som urtidsjægerne har levet i skovene og ved havet og helt frem til bondestenalderen. Gennem skeletfund afslører eksperter, hvordan vores forfædre har set ud i stenalderen og hvorfor stenaldermennesket byggede de imponerende jættestuer, som ligger overalt i det danske.',
	56	'timestamp': 1490401996,
	57	'upload_date': '20170325',
	58	'duration': 3502.04,
	59	},
5e9e3d0f	60	}]
f2b8db57 S	61
f2b8db57 S	62	def _real_extract(self, url):
6ad4013d	63	video_id = self._match_id(url)
f2b8db57	64
78271e33 S	65	webpage = self._download_webpage(url, video_id)
78271e33 S	66
aff84bec S	67	if '>Programmet er ikke længere tilgængeligt' in webpage:
	68	raise ExtractorError(
	69	'Video %s is not available' % video_id, expected=True)
	70
78271e33	71	video_id = self._search_regex(
5e9e3d0f SB	72	(r'data-(?:material-identifier\|episode-slug)="([^"]+)"',
5e9e3d0f SB	73	r'data-resource="[^>"]+mu/programcard/expanded/([^"]+)"'),
78271e33	74	webpage, 'video id')
f2b8db57	75
78271e33 S	76	programcard = self._download_json(
	77	'http://www.dr.dk/mu/programcard/expanded/%s' % video_id,
	78	video_id, 'Downloading video JSON')
f2b8db57 S	79	data = programcard['Data'][0]
f2b8db57 S	80
6066d03d S	81	title = remove_end(self._og_search_title(
	82	webpage, default=None), ' \| TV \| DR') or data['Title']
	83	description = self._og_search_description(
	84	webpage, default=None) or data.get('Description')
5e9e3d0f	85
6066d03d	86	timestamp = parse_iso8601(data.get('CreatedTime'))
f2b8db57 S	87
	88	thumbnail = None
	89	duration = None
	90
	91	restricted_to_denmark = False
	92
	93	formats = []
	94	subtitles = {}
	95
	96	for asset in data['Assets']:
2c15db82 RA	97	kind = asset.get('Kind')
2c15db82 RA	98	if kind == 'Image':
6066d03d	99	thumbnail = asset.get('Uri')
b972fb03 RR	100	preference = 0
	101
	102	sign_language = asset.get('Target') == 'SignLanguage'
	103
	104	if kind in ('VideoResource', 'AudioResource'):
6066d03d S	105	duration = float_or_none(asset.get('DurationInMilliseconds'), 1000)
	106	restricted_to_denmark = asset.get('RestrictedToDenmark')
	107	spoken_subtitles = asset.get('Target') == 'SpokenSubtitles'
	108	for link in asset.get('Links', []):
	109	uri = link.get('Uri')
	110	if not uri:
	111	continue
	112	target = link.get('Target')
	113	format_id = target or ''
6d2749aa	114	if spoken_subtitles:
1335c3ac	115	preference = -1
6d2749aa	116	format_id += '-spoken-subtitles'
b972fb03 RR	117	if sign_language:
	118	preference = -1
	119	format_id += "-sign-language"
1335c3ac	120	if target == 'HDS':
2c15db82	121	f4m_formats = self._extract_f4m_formats(
1335c3ac	122	uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43',
2c15db82 RA	123	video_id, preference, f4m_id=format_id)
	124	if kind == 'AudioResource':
	125	for f in f4m_formats:
	126	f['vcodec'] = 'none'
	127	formats.extend(f4m_formats)
1335c3ac S	128	elif target == 'HLS':
1335c3ac S	129	formats.extend(self._extract_m3u8_formats(
6066d03d S	130	uri, video_id, 'mp4', entry_protocol='m3u8_native',
6066d03d S	131	preference=preference, m3u8_id=format_id))
1335c3ac S	132	else:
	133	bitrate = link.get('Bitrate')
	134	if bitrate:
	135	format_id += '-%s' % bitrate
	136	formats.append({
	137	'url': uri,
	138	'format_id': format_id,
6066d03d	139	'tbr': int_or_none(bitrate),
1335c3ac	140	'ext': link.get('FileFormat'),
2c15db82	141	'vcodec': 'none' if kind == 'AudioResource' else None,
1335c3ac	142	})
f2b8db57 S	143	subtitles_list = asset.get('SubtitlesList')
	144	if isinstance(subtitles_list, list):
	145	LANGS = {
6fa73386	146	'Danish': 'da',
f2b8db57 S	147	}
f2b8db57 S	148	for subs in subtitles_list:
6066d03d S	149	if not subs.get('Uri'):
	150	continue
	151	lang = subs.get('Language') or 'da'
	152	subtitles.setdefault(LANGS.get(lang, lang), []).append({
	153	'url': subs['Uri'],
	154	'ext': mimetype2ext(subs.get('MimeType')) or 'vtt'
	155	})
f2b8db57 S	156
f2b8db57 S	157	if not formats and restricted_to_denmark:
6066d03d S	158	self.raise_geo_restricted(
6066d03d S	159	'Unfortunately, DR is not allowed to show this program outside Denmark.',
96182695	160	countries=self._GEO_COUNTRIES)
f2b8db57 S	161
	162	self._sort_formats(formats)
	163
f2b8db57 S	164	return {
	165	'id': video_id,
	166	'title': title,
	167	'description': description,
	168	'thumbnail': thumbnail,
	169	'timestamp': timestamp,
	170	'duration': duration,
	171	'formats': formats,
18c1c424	172	'subtitles': subtitles,
f2b8db57	173	}
2c15db82 RA	174
	175
	176	class DRTVLiveIE(InfoExtractor):
	177	IE_NAME = 'drtv:live'
	178	_VALID_URL = r'https?://(?:www\.)?dr\.dk/(?:tv\|TV)/live/(?P<id>[\da-z-]+)'
fc11ad38	179	_GEO_COUNTRIES = ['DK']
2c15db82 RA	180	_TEST = {
	181	'url': 'https://www.dr.dk/tv/live/dr1',
	182	'info_dict': {
	183	'id': 'dr1',
	184	'ext': 'mp4',
	185	'title': 're:^DR1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
	186	},
	187	'params': {
	188	# m3u8 download
	189	'skip_download': True,
	190	},
	191	}
	192
	193	def _real_extract(self, url):
	194	channel_id = self._match_id(url)
	195	channel_data = self._download_json(
	196	'https://www.dr.dk/mu-online/api/1.0/channel/' + channel_id,
	197	channel_id)
	198	title = self._live_title(channel_data['Title'])
	199
	200	formats = []
	201	for streaming_server in channel_data.get('StreamingServers', []):
	202	server = streaming_server.get('Server')
	203	if not server:
	204	continue
	205	link_type = streaming_server.get('LinkType')
	206	for quality in streaming_server.get('Qualities', []):
	207	for stream in quality.get('Streams', []):
	208	stream_path = stream.get('Stream')
	209	if not stream_path:
	210	continue
	211	stream_url = update_url_query(
	212	'%s/%s' % (server, stream_path), {'b': ''})
	213	if link_type == 'HLS':
	214	formats.extend(self._extract_m3u8_formats(
	215	stream_url, channel_id, 'mp4',
	216	m3u8_id=link_type, fatal=False, live=True))
	217	elif link_type == 'HDS':
	218	formats.extend(self._extract_f4m_formats(update_url_query(
	219	'%s/%s' % (server, stream_path), {'hdcore': '3.7.0'}),
	220	channel_id, f4m_id=link_type, fatal=False))
	221	self._sort_formats(formats)
	222
	223	return {
	224	'id': channel_id,
	225	'title': title,
	226	'thumbnail': channel_data.get('PrimaryImageUri'),
	227	'formats': formats,
	228	'is_live': True,
	229	}