[yt-dlp.git] / yt_dlp / extractor / adobetv.py

import functools
import re

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    ISO639Utils,
    OnDemandPagedList,
    float_or_none,
    int_or_none,
    join_nonempty,
    parse_duration,
    str_or_none,
    str_to_int,
    unified_strdate,
)


class AdobeTVBaseIE(InfoExtractor):
    def _call_api(self, path, video_id, query, note=None):
        return self._download_json(
            'http://tv.adobe.com/api/v4/' + path,
            video_id, note, query=query)['data']

    def _parse_subtitles(self, video_data, url_key):
        subtitles = {}
        for translation in video_data.get('translations', []):
            vtt_path = translation.get(url_key)
            if not vtt_path:
                continue
            lang = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
            subtitles.setdefault(lang, []).append({
                'ext': 'vtt',
                'url': vtt_path,
            })
        return subtitles

    def _parse_video_data(self, video_data):
        video_id = compat_str(video_data['id'])
        title = video_data['title']

        s3_extracted = False
        formats = []
        for source in video_data.get('videos', []):
            source_url = source.get('url')
            if not source_url:
                continue
            f = {
                'format_id': source.get('quality_level'),
                'fps': int_or_none(source.get('frame_rate')),
                'height': int_or_none(source.get('height')),
                'tbr': int_or_none(source.get('video_data_rate')),
                'width': int_or_none(source.get('width')),
                'url': source_url,
            }
            original_filename = source.get('original_filename')
            if original_filename:
                if not (f.get('height') and f.get('width')):
                    mobj = re.search(r'_(\d+)x(\d+)', original_filename)
                    if mobj:
                        f.update({
                            'height': int(mobj.group(2)),
                            'width': int(mobj.group(1)),
                        })
                if original_filename.startswith('s3://') and not s3_extracted:
                    formats.append({
                        'format_id': 'original',
                        'quality': 1,
                        'url': original_filename.replace('s3://', 'https://s3.amazonaws.com/'),
                    })
                    s3_extracted = True
            formats.append(f)

        return {
            'id': video_id,
            'title': title,
            'description': video_data.get('description'),
            'thumbnail': video_data.get('thumbnail'),
            'upload_date': unified_strdate(video_data.get('start_date')),
            'duration': parse_duration(video_data.get('duration')),
            'view_count': str_to_int(video_data.get('playcount')),
            'formats': formats,
            'subtitles': self._parse_subtitles(video_data, 'vtt'),
        }


class AdobeTVEmbedIE(AdobeTVBaseIE):
    IE_NAME = 'adobetv:embed'
    _VALID_URL = r'https?://tv\.adobe\.com/embed/\d+/(?P<id>\d+)'
    _TEST = {
        'url': 'https://tv.adobe.com/embed/22/4153',
        'md5': 'c8c0461bf04d54574fc2b4d07ac6783a',
        'info_dict': {
            'id': '4153',
            'ext': 'flv',
            'title': 'Creating Graphics Optimized for BlackBerry',
            'description': 'md5:eac6e8dced38bdaae51cd94447927459',
            'thumbnail': r're:https?://.*\.jpg$',
            'upload_date': '20091109',
            'duration': 377,
            'view_count': int,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_data = self._call_api(
            'episode/' + video_id, video_id, {'disclosure': 'standard'})[0]
        return self._parse_video_data(video_data)


class AdobeTVIE(AdobeTVBaseIE):
    IE_NAME = 'adobetv'
    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
        'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
        'info_dict': {
            'id': '10981',
            'ext': 'mp4',
            'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
            'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
            'thumbnail': r're:https?://.*\.jpg$',
            'upload_date': '20110914',
            'duration': 60,
            'view_count': int,
        },
    }

    def _real_extract(self, url):
        language, show_urlname, urlname = self._match_valid_url(url).groups()
        if not language:
            language = 'en'

        video_data = self._call_api(
            'episode/get', urlname, {
                'disclosure': 'standard',
                'language': language,
                'show_urlname': show_urlname,
                'urlname': urlname,
            })[0]
        return self._parse_video_data(video_data)


class AdobeTVPlaylistBaseIE(AdobeTVBaseIE):
    _PAGE_SIZE = 25

    def _fetch_page(self, display_id, query, page):
        page += 1
        query['page'] = page
        for element_data in self._call_api(
                self._RESOURCE, display_id, query, 'Download Page %d' % page):
            yield self._process_data(element_data)

    def _extract_playlist_entries(self, display_id, query):
        return OnDemandPagedList(functools.partial(
            self._fetch_page, display_id, query), self._PAGE_SIZE)


class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
    IE_NAME = 'adobetv:show'
    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?show/(?P<id>[^/]+)'

    _TEST = {
        'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost',
        'info_dict': {
            'id': '36',
            'title': 'The Complete Picture with Julieanne Kost',
            'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27',
        },
        'playlist_mincount': 136,
    }
    _RESOURCE = 'episode'
    _process_data = AdobeTVBaseIE._parse_video_data

    def _real_extract(self, url):
        language, show_urlname = self._match_valid_url(url).groups()
        if not language:
            language = 'en'
        query = {
            'disclosure': 'standard',
            'language': language,
            'show_urlname': show_urlname,
        }

        show_data = self._call_api(
            'show/get', show_urlname, query)[0]

        return self.playlist_result(
            self._extract_playlist_entries(show_urlname, query),
            str_or_none(show_data.get('id')),
            show_data.get('show_name'),
            show_data.get('show_description'))


class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
    IE_NAME = 'adobetv:channel'
    _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?channel/(?P<id>[^/]+)(?:/(?P<category_urlname>[^/]+))?'

    _TEST = {
        'url': 'http://tv.adobe.com/channel/development',
        'info_dict': {
            'id': 'development',
        },
        'playlist_mincount': 96,
    }
    _RESOURCE = 'show'

    def _process_data(self, show_data):
        return self.url_result(
            show_data['url'], 'AdobeTVShow', str_or_none(show_data.get('id')))

    def _real_extract(self, url):
        language, channel_urlname, category_urlname = self._match_valid_url(url).groups()
        if not language:
            language = 'en'
        query = {
            'channel_urlname': channel_urlname,
            'language': language,
        }
        if category_urlname:
            query['category_urlname'] = category_urlname

        return self.playlist_result(
            self._extract_playlist_entries(channel_urlname, query),
            channel_urlname)


class AdobeTVVideoIE(AdobeTVBaseIE):
    IE_NAME = 'adobetv:video'
    _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
    _EMBED_REGEX = [r'<iframe[^>]+src=[\'"](?P<url>(?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]']

    _TEST = {
        # From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
        'url': 'https://video.tv.adobe.com/v/2456/',
        'md5': '43662b577c018ad707a63766462b1e87',
        'info_dict': {
            'id': '2456',
            'ext': 'mp4',
            'title': 'New experience with Acrobat DC',
            'description': 'New experience with Acrobat DC',
            'duration': 248.667,
        },
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        video_data = self._parse_json(self._search_regex(
            r'var\s+bridge\s*=\s*([^;]+);', webpage, 'bridged data'), video_id)
        title = video_data['title']

        formats = []
        sources = video_data.get('sources') or []
        for source in sources:
            source_src = source.get('src')
            if not source_src:
                continue
            formats.append({
                'filesize': int_or_none(source.get('kilobytes') or None, invscale=1000),
                'format_id': join_nonempty(source.get('format'), source.get('label')),
                'height': int_or_none(source.get('height') or None),
                'tbr': int_or_none(source.get('bitrate') or None),
                'width': int_or_none(source.get('width') or None),
                'url': source_src,
            })

        # For both metadata and downloaded files the duration varies among
        # formats. I just pick the max one
        duration = max(filter(None, [
            float_or_none(source.get('duration'), scale=1000)
            for source in sources]))

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'description': video_data.get('description'),
            'thumbnail': video_data.get('video', {}).get('poster'),
            'duration': duration,
            'subtitles': self._parse_subtitles(video_data, 'vttPath'),
        }
Commit	Line	Data
7f641d2c	1	import functools
30bd1c16	2	import re
30bd1c16	3
1362bbbb	4	from .common import InfoExtractor
7079f8ff	5	from ..compat import compat_str
1362bbbb	6	from ..utils import (
e897bd82 SS	7	ISO639Utils,
e897bd82 SS	8	OnDemandPagedList,
607841af	9	float_or_none,
7f641d2c	10	int_or_none,
34921b43	11	join_nonempty,
7f641d2c RA	12	parse_duration,
	13	str_or_none,
	14	str_to_int,
	15	unified_strdate,
1362bbbb S	16	)
	17
	18
2c3b9f35	19	class AdobeTVBaseIE(InfoExtractor):
7f641d2c RA	20	def _call_api(self, path, video_id, query, note=None):
	21	return self._download_json(
	22	'http://tv.adobe.com/api/v4/' + path,
	23	video_id, note, query=query)['data']
	24
	25	def _parse_subtitles(self, video_data, url_key):
	26	subtitles = {}
	27	for translation in video_data.get('translations', []):
	28	vtt_path = translation.get(url_key)
	29	if not vtt_path:
	30	continue
	31	lang = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
	32	subtitles.setdefault(lang, []).append({
	33	'ext': 'vtt',
	34	'url': vtt_path,
	35	})
	36	return subtitles
	37
	38	def _parse_video_data(self, video_data):
	39	video_id = compat_str(video_data['id'])
	40	title = video_data['title']
	41
	42	s3_extracted = False
	43	formats = []
	44	for source in video_data.get('videos', []):
	45	source_url = source.get('url')
	46	if not source_url:
	47	continue
	48	f = {
	49	'format_id': source.get('quality_level'),
	50	'fps': int_or_none(source.get('frame_rate')),
	51	'height': int_or_none(source.get('height')),
	52	'tbr': int_or_none(source.get('video_data_rate')),
	53	'width': int_or_none(source.get('width')),
	54	'url': source_url,
	55	}
	56	original_filename = source.get('original_filename')
	57	if original_filename:
	58	if not (f.get('height') and f.get('width')):
	59	mobj = re.search(r'_(\d+)x(\d+)', original_filename)
	60	if mobj:
	61	f.update({
	62	'height': int(mobj.group(2)),
	63	'width': int(mobj.group(1)),
	64	})
	65	if original_filename.startswith('s3://') and not s3_extracted:
	66	formats.append({
	67	'format_id': 'original',
f983b875	68	'quality': 1,
7f641d2c RA	69	'url': original_filename.replace('s3://', 'https://s3.amazonaws.com/'),
	70	})
	71	s3_extracted = True
	72	formats.append(f)
7f641d2c RA	73
	74	return {
	75	'id': video_id,
	76	'title': title,
	77	'description': video_data.get('description'),
	78	'thumbnail': video_data.get('thumbnail'),
	79	'upload_date': unified_strdate(video_data.get('start_date')),
	80	'duration': parse_duration(video_data.get('duration')),
	81	'view_count': str_to_int(video_data.get('playcount')),
	82	'formats': formats,
	83	'subtitles': self._parse_subtitles(video_data, 'vtt'),
	84	}
	85
	86
	87	class AdobeTVEmbedIE(AdobeTVBaseIE):
	88	IE_NAME = 'adobetv:embed'
	89	_VALID_URL = r'https?://tv\.adobe\.com/embed/\d+/(?P<id>\d+)'
	90	_TEST = {
	91	'url': 'https://tv.adobe.com/embed/22/4153',
	92	'md5': 'c8c0461bf04d54574fc2b4d07ac6783a',
	93	'info_dict': {
	94	'id': '4153',
	95	'ext': 'flv',
	96	'title': 'Creating Graphics Optimized for BlackBerry',
	97	'description': 'md5:eac6e8dced38bdaae51cd94447927459',
	98	'thumbnail': r're:https?://.*\.jpg$',
	99	'upload_date': '20091109',
	100	'duration': 377,
	101	'view_count': int,
	102	},
	103	}
	104
	105	def _real_extract(self, url):
	106	video_id = self._match_id(url)
	107
	108	video_data = self._call_api(
	109	'episode/' + video_id, video_id, {'disclosure': 'standard'})[0]
	110	return self._parse_video_data(video_data)
2c3b9f35	111
	112
	113	class AdobeTVIE(AdobeTVBaseIE):
7f641d2c	114	IE_NAME = 'adobetv'
30bd1c16	115	_VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr\|de\|es\|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'
1362bbbb S	116
	117	_TEST = {
	118	'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
	119	'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
	120	'info_dict': {
30bd1c16	121	'id': '10981',
1362bbbb S	122	'ext': 'mp4',
	123	'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
	124	'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
ec85ded8	125	'thumbnail': r're:https?://.*\.jpg$',
1362bbbb S	126	'upload_date': '20110914',
	127	'duration': 60,
	128	'view_count': int,
	129	},
	130	}
	131
	132	def _real_extract(self, url):
5ad28e7f	133	language, show_urlname, urlname = self._match_valid_url(url).groups()
30bd1c16	134	if not language:
30bd1c16	135	language = 'en'
1362bbbb	136
7f641d2c RA	137	video_data = self._call_api(
	138	'episode/get', urlname, {
	139	'disclosure': 'standard',
	140	'language': language,
	141	'show_urlname': show_urlname,
	142	'urlname': urlname,
	143	})[0]
	144	return self._parse_video_data(video_data)
607841af YCH	145
607841af YCH	146
2c3b9f35	147	class AdobeTVPlaylistBaseIE(AdobeTVBaseIE):
7f641d2c RA	148	_PAGE_SIZE = 25
	149
	150	def _fetch_page(self, display_id, query, page):
	151	page += 1
	152	query['page'] = page
	153	for element_data in self._call_api(
	154	self._RESOURCE, display_id, query, 'Download Page %d' % page):
	155	yield self._process_data(element_data)
9a605c88	156
7f641d2c RA	157	def _extract_playlist_entries(self, display_id, query):
	158	return OnDemandPagedList(functools.partial(
	159	self._fetch_page, display_id, query), self._PAGE_SIZE)
9a605c88	160
	161
	162	class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
7f641d2c	163	IE_NAME = 'adobetv:show'
9a605c88	164	_VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr\|de\|es\|jp)/)?show/(?P<id>[^/]+)'
	165
	166	_TEST = {
	167	'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost',
	168	'info_dict': {
	169	'id': '36',
	170	'title': 'The Complete Picture with Julieanne Kost',
	171	'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27',
	172	},
	173	'playlist_mincount': 136,
	174	}
7f641d2c RA	175	_RESOURCE = 'episode'
7f641d2c RA	176	_process_data = AdobeTVBaseIE._parse_video_data
9a605c88	177
9a605c88	178	def _real_extract(self, url):
5ad28e7f	179	language, show_urlname = self._match_valid_url(url).groups()
9a605c88	180	if not language:
9a605c88	181	language = 'en'
7f641d2c RA	182	query = {
	183	'disclosure': 'standard',
	184	'language': language,
	185	'show_urlname': show_urlname,
	186	}
9a605c88	187
7f641d2c RA	188	show_data = self._call_api(
7f641d2c RA	189	'show/get', show_urlname, query)[0]
9a605c88	190
9a605c88	191	return self.playlist_result(
7f641d2c RA	192	self._extract_playlist_entries(show_urlname, query),
	193	str_or_none(show_data.get('id')),
	194	show_data.get('show_name'),
	195	show_data.get('show_description'))
9a605c88	196
	197
	198	class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
7f641d2c	199	IE_NAME = 'adobetv:channel'
9a605c88	200	_VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr\|de\|es\|jp)/)?channel/(?P<id>[^/]+)(?:/(?P<category_urlname>[^/]+))?'
	201
	202	_TEST = {
	203	'url': 'http://tv.adobe.com/channel/development',
	204	'info_dict': {
	205	'id': 'development',
	206	},
	207	'playlist_mincount': 96,
	208	}
7f641d2c	209	_RESOURCE = 'show'
9a605c88	210
7f641d2c RA	211	def _process_data(self, show_data):
	212	return self.url_result(
	213	show_data['url'], 'AdobeTVShow', str_or_none(show_data.get('id')))
9a605c88	214
9a605c88	215	def _real_extract(self, url):
5ad28e7f	216	language, channel_urlname, category_urlname = self._match_valid_url(url).groups()
9a605c88	217	if not language:
9a605c88	218	language = 'en'
7f641d2c RA	219	query = {
	220	'channel_urlname': channel_urlname,
	221	'language': language,
	222	}
9a605c88	223	if category_urlname:
7f641d2c	224	query['category_urlname'] = category_urlname
9a605c88	225
9a605c88	226	return self.playlist_result(
7f641d2c	227	self._extract_playlist_entries(channel_urlname, query),
9a605c88	228	channel_urlname)
	229
	230
7f641d2c RA	231	class AdobeTVVideoIE(AdobeTVBaseIE):
7f641d2c RA	232	IE_NAME = 'adobetv:video'
607841af	233	_VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
bfd973ec	234	_EMBED_REGEX = [r'<iframe[^>]+src=[\'"](?P<url>(?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]']
607841af YCH	235
607841af YCH	236	_TEST = {
a5158f38	237	# From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
607841af YCH	238	'url': 'https://video.tv.adobe.com/v/2456/',
	239	'md5': '43662b577c018ad707a63766462b1e87',
	240	'info_dict': {
	241	'id': '2456',
	242	'ext': 'mp4',
	243	'title': 'New experience with Acrobat DC',
	244	'description': 'New experience with Acrobat DC',
	245	'duration': 248.667,
	246	},
	247	}
	248
	249	def _real_extract(self, url):
	250	video_id = self._match_id(url)
26264cb0 YCH	251	webpage = self._download_webpage(url, video_id)
	252
	253	video_data = self._parse_json(self._search_regex(
	254	r'var\s+bridge\s=\s([^;]+);', webpage, 'bridged data'), video_id)
7f641d2c RA	255	title = video_data['title']
	256
	257	formats = []
	258	sources = video_data.get('sources') or []
	259	for source in sources:
	260	source_src = source.get('src')
	261	if not source_src:
	262	continue
	263	formats.append({
	264	'filesize': int_or_none(source.get('kilobytes') or None, invscale=1000),
34921b43	265	'format_id': join_nonempty(source.get('format'), source.get('label')),
7f641d2c RA	266	'height': int_or_none(source.get('height') or None),
	267	'tbr': int_or_none(source.get('bitrate') or None),
	268	'width': int_or_none(source.get('width') or None),
	269	'url': source_src,
	270	})
607841af YCH	271
	272	# For both metadata and downloaded files the duration varies among
	273	# formats. I just pick the max one
	274	duration = max(filter(None, [
	275	float_or_none(source.get('duration'), scale=1000)
7f641d2c	276	for source in sources]))
607841af YCH	277
	278	return {
	279	'id': video_id,
	280	'formats': formats,
7f641d2c	281	'title': title,
402ca40c	282	'description': video_data.get('description'),
7f641d2c	283	'thumbnail': video_data.get('video', {}).get('poster'),
607841af	284	'duration': duration,
7f641d2c	285	'subtitles': self._parse_subtitles(video_data, 'vttPath'),
607841af	286	}