[yt-dlp.git] / youtube_dl / extractor / wdr.py

# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    ExtractorError,
    js_to_json,
    strip_jsonp,
    unified_strdate,
    update_url_query,
    urlhandle_detect_ext,
)


class WDRBaseIE(InfoExtractor):
    def _extract_wdr_video(self, webpage, display_id):
        # for wdr.de the data-extension is in a tag with the class "mediaLink"
        # for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
        # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
        json_metadata = self._html_search_regex(
            r'class=(?:"(?:mediaLink|wdrrPlayerPlayBtn)\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
            webpage, 'media link', default=None, flags=re.MULTILINE)

        if not json_metadata:
            return

        media_link_obj = self._parse_json(json_metadata, display_id,
                                          transform_source=js_to_json)
        jsonp_url = media_link_obj['mediaObj']['url']

        metadata = self._download_json(
            jsonp_url, 'metadata', transform_source=strip_jsonp)

        metadata_tracker_data = metadata['trackerData']
        metadata_media_resource = metadata['mediaResource']

        formats = []

        # check if the metadata contains a direct URL to a file
        for kind, media_resource in metadata_media_resource.items():
            if kind not in ('dflt', 'alt'):
                continue

            for tag_name, medium_url in media_resource.items():
                if tag_name not in ('videoURL', 'audioURL'):
                    continue

                ext = determine_ext(medium_url)
                if ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
                        medium_url, display_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls'))
                elif ext == 'f4m':
                    manifest_url = update_url_query(
                        medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
                    formats.extend(self._extract_f4m_formats(
                        manifest_url, display_id, f4m_id='hds', fatal=False))
                elif ext == 'smil':
                    formats.extend(self._extract_smil_formats(
                        medium_url, 'stream', fatal=False))
                else:
                    a_format = {
                        'url': medium_url
                    }
                    if ext == 'unknown_video':
                        urlh = self._request_webpage(
                            medium_url, display_id, note='Determining extension')
                        ext = urlhandle_detect_ext(urlh)
                        a_format['ext'] = ext
                    formats.append(a_format)

        self._sort_formats(formats)

        subtitles = {}
        caption_url = metadata_media_resource.get('captionURL')
        if caption_url:
            subtitles['de'] = [{
                'url': caption_url,
                'ext': 'ttml',
            }]

        title = metadata_tracker_data['trackerClipTitle']

        return {
            'id': metadata_tracker_data.get('trackerClipId', display_id),
            'display_id': display_id,
            'title': title,
            'alt_title': metadata_tracker_data.get('trackerClipSubcategory'),
            'formats': formats,
            'subtitles': subtitles,
            'upload_date': unified_strdate(metadata_tracker_data.get('trackerClipAirTime')),
        }


class WDRIE(WDRBaseIE):
    _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
    _PAGE_REGEX = r'/(?:mediathek/)?[^/]+/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
    _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL

    _TESTS = [
        {
            'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
            # HDS download, MD5 is unstable
            'info_dict': {
                'id': 'mdb-1058683',
                'ext': 'flv',
                'display_id': 'doku-am-freitag/video-geheimnis-aachener-dom-100',
                'title': 'Geheimnis Aachener Dom',
                'alt_title': 'Doku am Freitag',
                'upload_date': '20160304',
                'description': 'md5:87be8ff14d8dfd7a7ee46f0299b52318',
                'is_live': False,
                'subtitles': {'de': [{
                    'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml',
                    'ext': 'ttml',
                }]},
            },
        },
        {
            'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html',
            'md5': 'f4c1f96d01cf285240f53ea4309663d8',
            'info_dict': {
                'id': 'mdb-1072000',
                'ext': 'mp3',
                'display_id': 'wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100',
                'title': 'Schriftstellerin Juli Zeh',
                'alt_title': 'WDR 3 Gespräch am Samstag',
                'upload_date': '20160312',
                'description': 'md5:e127d320bc2b1f149be697ce044a3dd7',
                'is_live': False,
                'subtitles': {}
            },
        },
        {
            'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
            'info_dict': {
                'id': 'mdb-103364',
                'ext': 'mp4',
                'display_id': 'index',
                'title': r're:^WDR Fernsehen im Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                'alt_title': 'WDR Fernsehen Live',
                'upload_date': None,
                'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
                'is_live': True,
                'subtitles': {}
            },
            'params': {
                'skip_download': True,  # m3u8 download
            },
        },
        {
            'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
            'playlist_mincount': 8,
            'info_dict': {
                'id': 'aktuelle-stunde/aktuelle-stunde-120',
            },
        },
        {
            'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
            'info_dict': {
                'id': 'mdb-1096487',
                'ext': 'flv',
                'upload_date': 're:^[0-9]{8}$',
                'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$',
                'description': '- Die Sendung mit der Maus -',
            },
            'skip': 'The id changes from week to week because of the new episode'
        },
        {
            'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
            'md5': '803138901f6368ee497b4d195bb164f2',
            'info_dict': {
                'id': 'mdb-186083',
                'ext': 'mp4',
                'upload_date': '20130919',
                'title': 'Sachgeschichte - Achterbahn ',
                'description': '- Die Sendung mit der Maus -',
            },
        },
        {
            'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html',
            # Live stream, MD5 unstable
            'info_dict': {
                'id': 'mdb-869971',
                'ext': 'flv',
                'title': 'Funkhaus Europa Livestream',
                'description': 'md5:2309992a6716c347891c045be50992e4',
                'upload_date': '20160101',
            },
        }
    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        url_type = mobj.group('type')
        page_url = mobj.group('page_url')
        display_id = mobj.group('display_id')
        webpage = self._download_webpage(url, display_id)

        info_dict = self._extract_wdr_video(webpage, display_id)

        if not info_dict:
            entries = [
                self.url_result(page_url + href[0], 'WDR')
                for href in re.findall(
                    r'<a href="(%s)"[^>]+data-extension=' % self._PAGE_REGEX,
                    webpage)
            ]

            if entries:  # Playlist page
                return self.playlist_result(entries, playlist_id=display_id)

            raise ExtractorError('No downloadable streams found', expected=True)

        is_live = url_type == 'live'

        if is_live:
            info_dict.update({
                'title': self._live_title(info_dict['title']),
                'upload_date': None,
            })
        elif 'upload_date' not in info_dict:
            info_dict['upload_date'] = unified_strdate(self._html_search_meta('DC.Date', webpage, 'upload date'))

        info_dict.update({
            'description': self._html_search_meta('Description', webpage),
            'is_live': is_live,
        })

        return info_dict


class WDRMobileIE(InfoExtractor):
    _VALID_URL = r'''(?x)
        https?://mobile-ondemand\.wdr\.de/
        .*?/fsk(?P<age_limit>[0-9]+)
        /[0-9]+/[0-9]+/
        (?P<id>[0-9]+)_(?P<title>[0-9]+)'''
    IE_NAME = 'wdr:mobile'
    _TEST = {
        'url': 'http://mobile-ondemand.wdr.de/CMS2010/mdb/ondemand/weltweit/fsk0/42/421735/421735_4283021.mp4',
        'info_dict': {
            'title': '4283021',
            'id': '421735',
            'ext': 'mp4',
            'age_limit': 0,
        },
        'skip': 'Problems with loading data.'
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        return {
            'id': mobj.group('id'),
            'title': mobj.group('title'),
            'age_limit': int(mobj.group('age_limit')),
            'url': url,
            'http_headers': {
                'User-Agent': 'mobile',
            },
        }
Commit	Line	Data
dcdb292f	1	# coding: utf-8
b461641f S	2	from __future__ import unicode_literals
	3
	4	import re
	5
	6	from .common import InfoExtractor
1cc79574	7	from ..utils import (
bec2c14f	8	determine_ext,
1594a493	9	ExtractorError,
33a1ff71	10	js_to_json,
37f97295	11	strip_jsonp,
cd7481a3	12	unified_strdate,
6869d634	13	update_url_query,
50918c4e	14	urlhandle_detect_ext,
b461641f S	15	)
	16
	17
6d0d4fc2 YCH	18	class WDRBaseIE(InfoExtractor):
	19	def _extract_wdr_video(self, webpage, display_id):
	20	# for wdr.de the data-extension is in a tag with the class "mediaLink"
	21	# for wdr.de radio players, in a tag with the class "wdrrPlayerPlayBtn"
	22	# for wdrmaus its in a link to the page in a multiline "videoLink"-tag
	23	json_metadata = self._html_search_regex(
	24	r'class=(?:"(?:mediaLink\|wdrrPlayerPlayBtn)\b[^"]"[^>]+\|"videoLink\b[^"]"[\s]>\n[^\n])data-extension="([^"]+)"',
	25	webpage, 'media link', default=None, flags=re.MULTILINE)
	26
	27	if not json_metadata:
	28	return
	29
	30	media_link_obj = self._parse_json(json_metadata, display_id,
	31	transform_source=js_to_json)
	32	jsonp_url = media_link_obj['mediaObj']['url']
	33
	34	metadata = self._download_json(
	35	jsonp_url, 'metadata', transform_source=strip_jsonp)
	36
	37	metadata_tracker_data = metadata['trackerData']
	38	metadata_media_resource = metadata['mediaResource']
	39
	40	formats = []
	41
	42	# check if the metadata contains a direct URL to a file
	43	for kind, media_resource in metadata_media_resource.items():
	44	if kind not in ('dflt', 'alt'):
	45	continue
	46
	47	for tag_name, medium_url in media_resource.items():
	48	if tag_name not in ('videoURL', 'audioURL'):
	49	continue
	50
	51	ext = determine_ext(medium_url)
	52	if ext == 'm3u8':
	53	formats.extend(self._extract_m3u8_formats(
	54	medium_url, display_id, 'mp4', 'm3u8_native',
	55	m3u8_id='hls'))
	56	elif ext == 'f4m':
	57	manifest_url = update_url_query(
	58	medium_url, {'hdcore': '3.2.0', 'plugin': 'aasp-3.2.0.77.18'})
	59	formats.extend(self._extract_f4m_formats(
	60	manifest_url, display_id, f4m_id='hds', fatal=False))
	61	elif ext == 'smil':
	62	formats.extend(self._extract_smil_formats(
	63	medium_url, 'stream', fatal=False))
	64	else:
	65	a_format = {
	66	'url': medium_url
	67	}
	68	if ext == 'unknown_video':
	69	urlh = self._request_webpage(
	70	medium_url, display_id, note='Determining extension')
	71	ext = urlhandle_detect_ext(urlh)
	72	a_format['ext'] = ext
	73	formats.append(a_format)
	74
	75	self._sort_formats(formats)
	76
	77	subtitles = {}
	78	caption_url = metadata_media_resource.get('captionURL')
	79	if caption_url:
	80	subtitles['de'] = [{
	81	'url': caption_url,
82	'ext': 'ttml',
83	}]
84
85	title = metadata_tracker_data['trackerClipTitle']
86
87	return {
88	'id': metadata_tracker_data.get('trackerClipId', display_id),
89	'display_id': display_id,
90	'title': title,
91	'alt_title': metadata_tracker_data.get('trackerClipSubcategory'),
92	'formats': formats,
93	'subtitles': subtitles,
94	'upload_date': unified_strdate(metadata_tracker_data.get('trackerClipAirTime')),
95	}
96
97
98	class WDRIE(WDRBaseIE):
949fc42e	99	_CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
1594a493	100	_PAGE_REGEX = r'/(?:mediathek/)?[^/]+/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
3874e6ea	101	_VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '\|' + _CURRENT_MAUS_URL
c0837a12	102
b461641f S	103	_TESTS = [
b461641f S	104	{
c0837a12	105	'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
a4a8201c	106	# HDS download, MD5 is unstable
b461641f	107	'info_dict': {
c0837a12	108	'id': 'mdb-1058683',
b461641f	109	'ext': 'flv',
c0837a12 BW	110	'display_id': 'doku-am-freitag/video-geheimnis-aachener-dom-100',
	111	'title': 'Geheimnis Aachener Dom',
	112	'alt_title': 'Doku am Freitag',
	113	'upload_date': '20160304',
	114	'description': 'md5:87be8ff14d8dfd7a7ee46f0299b52318',
	115	'is_live': False,
	116	'subtitles': {'de': [{
bc7e7adf YCH	117	'url': 'http://ondemand-ww.wdr.de/medp/fsk0/105/1058683/1058683_12220974.xml',
bc7e7adf YCH	118	'ext': 'ttml',
c0837a12	119	}]},
b461641f S	120	},
	121	},
	122	{
c0837a12 BW	123	'url': 'http://www1.wdr.de/mediathek/audio/wdr3/wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100.html',
c0837a12 BW	124	'md5': 'f4c1f96d01cf285240f53ea4309663d8',
b461641f	125	'info_dict': {
c0837a12	126	'id': 'mdb-1072000',
b461641f	127	'ext': 'mp3',
c0837a12 BW	128	'display_id': 'wdr3-gespraech-am-samstag/audio-schriftstellerin-juli-zeh-100',
	129	'title': 'Schriftstellerin Juli Zeh',
	130	'alt_title': 'WDR 3 Gespräch am Samstag',
	131	'upload_date': '20160312',
	132	'description': 'md5:e127d320bc2b1f149be697ce044a3dd7',
	133	'is_live': False,
	134	'subtitles': {}
b461641f S	135	},
b461641f S	136	},
176cf9e0	137	{
c0837a12	138	'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
dd8982f1	139	'info_dict': {
c0837a12	140	'id': 'mdb-103364',
bec2c14f	141	'ext': 'mp4',
c0837a12 BW	142	'display_id': 'index',
	143	'title': r're:^WDR Fernsehen im Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
	144	'alt_title': 'WDR Fernsehen Live',
	145	'upload_date': None,
	146	'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
	147	'is_live': True,
	148	'subtitles': {}
bec2c14f BW	149	},
	150	'params': {
	151	'skip_download': True, # m3u8 download
	152	},
b8988b63 AA	153	},
b8988b63 AA	154	{
c0837a12	155	'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
3a686853	156	'playlist_mincount': 8,
b8988b63	157	'info_dict': {
c0837a12	158	'id': 'aktuelle-stunde/aktuelle-stunde-120',
b8988b63	159	},
14f7a2b8 BW	160	},
	161	{
	162	'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
	163	'info_dict': {
	164	'id': 'mdb-1096487',
	165	'ext': 'flv',
	166	'upload_date': 're:^[0-9]{8}$',
	167	'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$',
	168	'description': '- Die Sendung mit der Maus -',
	169	},
	170	'skip': 'The id changes from week to week because of the new episode'
	171	},
949fc42e BW	172	{
949fc42e BW	173	'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
6d0d4fc2	174	'md5': '803138901f6368ee497b4d195bb164f2',
949fc42e BW	175	'info_dict': {
949fc42e BW	176	'id': 'mdb-186083',
6d0d4fc2	177	'ext': 'mp4',
949fc42e BW	178	'upload_date': '20130919',
	179	'title': 'Sachgeschichte - Achterbahn ',
	180	'description': '- Die Sendung mit der Maus -',
	181	},
949fc42e	182	},
50918c4e YCH	183	{
50918c4e YCH	184	'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html',
6869d634	185	# Live stream, MD5 unstable
50918c4e YCH	186	'info_dict': {
50918c4e YCH	187	'id': 'mdb-869971',
6869d634	188	'ext': 'flv',
50918c4e YCH	189	'title': 'Funkhaus Europa Livestream',
	190	'description': 'md5:2309992a6716c347891c045be50992e4',
	191	'upload_date': '20160101',
	192	},
	193	}
b461641f S	194	]
	195
	196	def _real_extract(self, url):
	197	mobj = re.match(self._VALID_URL, url)
c0837a12 BW	198	url_type = mobj.group('type')
	199	page_url = mobj.group('page_url')
	200	display_id = mobj.group('display_id')
	201	webpage = self._download_webpage(url, display_id)
b461641f	202
6d0d4fc2	203	info_dict = self._extract_wdr_video(webpage, display_id)
b461641f	204
6d0d4fc2	205	if not info_dict:
b461641f	206	entries = [
c0837a12	207	self.url_result(page_url + href[0], 'WDR')
03681819	208	for href in re.findall(
3a686853	209	r'<a href="(%s)"[^>]+data-extension=' % self._PAGE_REGEX,
03681819	210	webpage)
b461641f	211	]
176cf9e0 PH	212
176cf9e0 PH	213	if entries: # Playlist page
c0837a12	214	return self.playlist_result(entries, playlist_id=display_id)
176cf9e0	215
c0837a12 BW	216	raise ExtractorError('No downloadable streams found', expected=True)
c0837a12 BW	217
c0837a12	218	is_live = url_type == 'live'
b8988b63 AA	219
b8988b63 AA	220	if is_live:
6d0d4fc2 YCH	221	info_dict.update({
	222	'title': self._live_title(info_dict['title']),
	223	'upload_date': None,
	224	})
	225	elif 'upload_date' not in info_dict:
	226	info_dict['upload_date'] = unified_strdate(self._html_search_meta('DC.Date', webpage, 'upload date'))
b461641f	227
6d0d4fc2	228	info_dict.update({
3874e6ea	229	'description': self._html_search_meta('Description', webpage),
c0837a12	230	'is_live': is_live,
6d0d4fc2 YCH	231	})
	232
	233	return info_dict
cd7481a3 PH	234
cd7481a3 PH	235
e4cbb5f3 PH	236	class WDRMobileIE(InfoExtractor):
	237	_VALID_URL = r'''(?x)
	238	https?://mobile-ondemand\.wdr\.de/
	239	.*?/fsk(?P<age_limit>[0-9]+)
	240	/[0-9]+/[0-9]+/
	241	(?P<id>[0-9]+)_(?P<title>[0-9]+)'''
	242	IE_NAME = 'wdr:mobile'
	243	_TEST = {
	244	'url': 'http://mobile-ondemand.wdr.de/CMS2010/mdb/ondemand/weltweit/fsk0/42/421735/421735_4283021.mp4',
	245	'info_dict': {
	246	'title': '4283021',
	247	'id': '421735',
7807ee66	248	'ext': 'mp4',
e4cbb5f3 PH	249	'age_limit': 0,
e4cbb5f3 PH	250	},
7807ee66	251	'skip': 'Problems with loading data.'
e4cbb5f3 PH	252	}
	253
	254	def _real_extract(self, url):
	255	mobj = re.match(self._VALID_URL, url)
	256	return {
	257	'id': mobj.group('id'),
	258	'title': mobj.group('title'),
	259	'age_limit': int(mobj.group('age_limit')),
	260	'url': url,
e1554a40 JMF	261	'http_headers': {
	262	'User-Agent': 'mobile',
	263	},
e4cbb5f3	264	}