[yt-dlp.git] / youtube_dl / extractor / appletrailers.py

from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
)


class AppleTrailersIE(InfoExtractor):
    IE_NAME = 'appletrailers'
    _VALID_URL = r'https?://(?:www\.|movie)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
    _TESTS = [{
        'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
        'info_dict': {
            'id': 'manofsteel',
        },
        'playlist': [
            {
                'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
                'info_dict': {
                    'id': 'manofsteel-trailer4',
                    'ext': 'mov',
                    'duration': 111,
                    'title': 'Trailer 4',
                    'upload_date': '20130523',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
                'info_dict': {
                    'id': 'manofsteel-trailer3',
                    'ext': 'mov',
                    'duration': 182,
                    'title': 'Trailer 3',
                    'upload_date': '20130417',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'd0f1e1150989b9924679b441f3404d48',
                'info_dict': {
                    'id': 'manofsteel-trailer',
                    'ext': 'mov',
                    'duration': 148,
                    'title': 'Trailer',
                    'upload_date': '20121212',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': '5fe08795b943eb2e757fa95cb6def1cb',
                'info_dict': {
                    'id': 'manofsteel-teaser',
                    'ext': 'mov',
                    'duration': 93,
                    'title': 'Teaser',
                    'upload_date': '20120721',
                    'uploader_id': 'wb',
                },
            },
        ]
    }, {
        'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
        'info_dict': {
            'id': 'blackthorn',
        },
        'playlist_mincount': 2,
    }, {
        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
        'only_matching': True,
    }, {
        'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',
        'only_matching': True,
    }]

    _JSON_RE = r'iTunes.playURL\((.*?)\);'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

        playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')

        def fix_html(s):
            s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
            s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)
            # The ' in the onClick attributes are not escaped, it couldn't be parsed
            # like: http://trailers.apple.com/trailers/wb/gravity/

            def _clean_json(m):
                return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
            s = re.sub(self._JSON_RE, _clean_json, s)
            s = '<html>%s</html>' % s
            return s
        doc = self._download_xml(playlist_url, movie, transform_source=fix_html)

        playlist = []
        for li in doc.findall('./div/ul/li'):
            on_click = li.find('.//a').attrib['onClick']
            trailer_info_json = self._search_regex(self._JSON_RE,
                                                   on_click, 'trailer info')
            trailer_info = json.loads(trailer_info_json)
            first_url = trailer_info.get('url')
            if not first_url:
                continue
            title = trailer_info['title']
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']
            upload_date = trailer_info['posted'].replace('-', '')

            runtime = trailer_info['runtime']
            m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

            trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
            settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
            settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

            formats = []
            for format in settings['metadata']['sizes']:
                # The src is a file pointing to the real video file
                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
                formats.append({
                    'url': format_url,
                    'format': format['type'],
                    'width': int_or_none(format['width']),
                    'height': int_or_none(format['height']),
                })

            self._sort_formats(formats)

            playlist.append({
                '_type': 'video',
                'id': video_id,
                'formats': formats,
                'title': title,
                'duration': duration,
                'thumbnail': thumbnail,
                'upload_date': upload_date,
                'uploader_id': uploader_id,
                'http_headers': {
                    'User-Agent': 'QuickTime compatible (youtube-dl)',
                },
            })

        return {
            '_type': 'playlist',
            'id': movie,
            'entries': playlist,
        }


class AppleTrailersSectionIE(InfoExtractor):
    IE_NAME = 'appletrailers:section'
    _SECTIONS = {
        'justadded': {
            'feed_path': 'just_added',
            'title': 'Just Added',
        },
        'exclusive': {
            'feed_path': 'exclusive',
            'title': 'Exclusive',
        },
        'justhd': {
            'feed_path': 'just_hd',
            'title': 'Just HD',
        },
        'mostpopular': {
            'feed_path': 'most_pop',
            'title': 'Most Popular',
        },
        'moviestudios': {
            'feed_path': 'studios',
            'title': 'Movie Studios',
        },
    }
    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '|'.join(_SECTIONS)
    _TESTS = [{
        'url': 'http://trailers.apple.com/#section=justadded',
        'info_dict': {
            'title': 'Just Added',
            'id': 'justadded',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=exclusive',
        'info_dict': {
            'title': 'Exclusive',
            'id': 'exclusive',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=justhd',
        'info_dict': {
            'title': 'Just HD',
            'id': 'justhd',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=mostpopular',
        'info_dict': {
            'title': 'Most Popular',
            'id': 'mostpopular',
        },
        'playlist_mincount': 80,
    }, {
        'url': 'http://trailers.apple.com/#section=moviestudios',
        'info_dict': {
            'title': 'Movie Studios',
            'id': 'moviestudios',
        },
        'playlist_mincount': 80,
    }]

    def _real_extract(self, url):
        section = self._match_id(url)
        section_data = self._download_json(
            'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],
            section)
        entries = [
            self.url_result('http://trailers.apple.com' + e['location'])
            for e in section_data]
        return self.playlist_result(entries, section, self._SECTIONS[section]['title'])
Commit	Line	Data
3798eadc PH	1	from __future__ import unicode_literals
3798eadc PH	2
44586389	3	import re
84353056	4	import json
44586389 PH	5
44586389 PH	6	from .common import InfoExtractor
1cc79574	7	from ..compat import compat_urlparse
44586389	8	from ..utils import (
9572013d	9	int_or_none,
44586389 PH	10	)
	11
	12
	13	class AppleTrailersIE(InfoExtractor):
60427f63	14	IE_NAME = 'appletrailers'
958759f4	15	_VALID_URL = r'https?://(?:www\.\|movie)?trailers\.apple\.com/(?:trailers\|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
35b79823	16	_TESTS = [{
e759a001	17	'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
11e611a7 PH	18	'info_dict': {
	19	'id': 'manofsteel',
	20	},
e759a001	21	'playlist': [
44586389	22	{
e759a001 S	23	'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
	24	'info_dict': {
	25	'id': 'manofsteel-trailer4',
	26	'ext': 'mov',
	27	'duration': 111,
	28	'title': 'Trailer 4',
	29	'upload_date': '20130523',
	30	'uploader_id': 'wb',
44586389 PH	31	},
	32	},
	33	{
e759a001 S	34	'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
	35	'info_dict': {
	36	'id': 'manofsteel-trailer3',
	37	'ext': 'mov',
	38	'duration': 182,
	39	'title': 'Trailer 3',
	40	'upload_date': '20130417',
	41	'uploader_id': 'wb',
44586389 PH	42	},
	43	},
	44	{
e759a001 S	45	'md5': 'd0f1e1150989b9924679b441f3404d48',
	46	'info_dict': {
	47	'id': 'manofsteel-trailer',
	48	'ext': 'mov',
	49	'duration': 148,
	50	'title': 'Trailer',
	51	'upload_date': '20121212',
	52	'uploader_id': 'wb',
44586389 PH	53	},
	54	},
	55	{
e759a001 S	56	'md5': '5fe08795b943eb2e757fa95cb6def1cb',
	57	'info_dict': {
	58	'id': 'manofsteel-teaser',
	59	'ext': 'mov',
	60	'duration': 93,
	61	'title': 'Teaser',
	62	'upload_date': '20120721',
	63	'uploader_id': 'wb',
44586389	64	},
28acf550	65	},
44586389	66	]
261b4c23	67	}, {
	68	'url': 'http://trailers.apple.com/trailers/magnolia/blackthorn/',
	69	'info_dict': {
	70	'id': 'blackthorn',
	71	},
	72	'playlist_mincount': 2,
35b79823 S	73	}, {
	74	'url': 'http://trailers.apple.com/ca/metropole/autrui/',
	75	'only_matching': True,
958759f4 YCH	76	}, {
	77	'url': 'http://movietrailers.apple.com/trailers/focus_features/kuboandthetwostrings/',
	78	'only_matching': True,
35b79823	79	}]
44586389	80
84353056 JMF	81	_JSON_RE = r'iTunes.playURL\((.*?)\);'
84353056 JMF	82
44586389 PH	83	def _real_extract(self, url):
	84	mobj = re.match(self._VALID_URL, url)
	85	movie = mobj.group('movie')
	86	uploader_id = mobj.group('company')
	87
28acf550	88	playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
5f6a1245	89
18258362	90	def fix_html(s):
28acf550	91	s = re.sub(r'(?s)<script[^<]?>.?</script>', '', s)
7fe37d8a	92	s = re.sub(r'<img ([^<]*?)/?>', r'<img \1/>', s)
18258362 JMF	93	# The ' in the onClick attributes are not escaped, it couldn't be parsed
18258362 JMF	94	# like: http://trailers.apple.com/trailers/wb/gravity/
5f6a1245	95
18258362	96	def _clean_json(m):
28acf550	97	return 'iTunes.playURL(%s);' % m.group(1).replace('\'', ''')
18258362	98	s = re.sub(self._JSON_RE, _clean_json, s)
e91cdcae	99	s = '<html>%s</html>' % s
18258362 JMF	100	return s
18258362 JMF	101	doc = self._download_xml(playlist_url, movie, transform_source=fix_html)
44586389	102
44586389 PH	103	playlist = []
44586389 PH	104	for li in doc.findall('./div/ul/li'):
84353056 JMF	105	on_click = li.find('.//a').attrib['onClick']
84353056 JMF	106	trailer_info_json = self._search_regex(self._JSON_RE,
9e1a5b84	107	on_click, 'trailer info')
84353056	108	trailer_info = json.loads(trailer_info_json)
261b4c23	109	first_url = trailer_info.get('url')
	110	if not first_url:
	111	continue
84353056	112	title = trailer_info['title']
44586389 PH	113	video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
44586389 PH	114	thumbnail = li.find('.//img').attrib['src']
84353056	115	upload_date = trailer_info['posted'].replace('-', '')
44586389	116
84353056 JMF	117	runtime = trailer_info['runtime']
84353056 JMF	118	m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
44586389 PH	119	duration = None
	120	if m:
	121	duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
	122
bb4aa62c	123	trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
84353056	124	settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
28acf550	125	settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')
44586389	126
84353056 JMF	127	formats = []
	128	for format in settings['metadata']['sizes']:
	129	# The src is a file pointing to the real video file
	130	format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
	131	formats.append({
	132	'url': format_url,
84353056	133	'format': format['type'],
9572013d PH	134	'width': int_or_none(format['width']),
9572013d PH	135	'height': int_or_none(format['height']),
84353056	136	})
7b8af563 PH	137
7b8af563 PH	138	self._sort_formats(formats)
44586389	139
fb7abb31	140	playlist.append({
44586389 PH	141	'_type': 'video',
44586389 PH	142	'id': video_id,
44586389 PH	143	'formats': formats,
	144	'title': title,
	145	'duration': duration,
	146	'thumbnail': thumbnail,
	147	'upload_date': upload_date,
	148	'uploader_id': uploader_id,
e1554a40 JMF	149	'http_headers': {
	150	'User-Agent': 'QuickTime compatible (youtube-dl)',
	151	},
fb7abb31	152	})
44586389 PH	153
	154	return {
	155	'_type': 'playlist',
	156	'id': movie,
	157	'entries': playlist,
	158	}
60427f63	159
	160
	161	class AppleTrailersSectionIE(InfoExtractor):
	162	IE_NAME = 'appletrailers:section'
	163	_SECTIONS = {
	164	'justadded': {
	165	'feed_path': 'just_added',
	166	'title': 'Just Added',
	167	},
	168	'exclusive': {
	169	'feed_path': 'exclusive',
	170	'title': 'Exclusive',
	171	},
	172	'justhd': {
	173	'feed_path': 'just_hd',
	174	'title': 'Just HD',
	175	},
	176	'mostpopular': {
	177	'feed_path': 'most_pop',
	178	'title': 'Most Popular',
	179	},
	180	'moviestudios': {
	181	'feed_path': 'studios',
	182	'title': 'Movie Studios',
	183	},
	184	}
	185	_VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/#section=(?P<id>%s)' % '\|'.join(_SECTIONS)
	186	_TESTS = [{
	187	'url': 'http://trailers.apple.com/#section=justadded',
	188	'info_dict': {
	189	'title': 'Just Added',
	190	'id': 'justadded',
	191	},
	192	'playlist_mincount': 80,
	193	}, {
	194	'url': 'http://trailers.apple.com/#section=exclusive',
	195	'info_dict': {
	196	'title': 'Exclusive',
	197	'id': 'exclusive',
	198	},
	199	'playlist_mincount': 80,
	200	}, {
	201	'url': 'http://trailers.apple.com/#section=justhd',
	202	'info_dict': {
	203	'title': 'Just HD',
	204	'id': 'justhd',
	205	},
	206	'playlist_mincount': 80,
	207	}, {
	208	'url': 'http://trailers.apple.com/#section=mostpopular',
	209	'info_dict': {
	210	'title': 'Most Popular',
	211	'id': 'mostpopular',
	212	},
	213	'playlist_mincount': 80,
	214	}, {
	215	'url': 'http://trailers.apple.com/#section=moviestudios',
	216	'info_dict': {
	217	'title': 'Movie Studios',
	218	'id': 'moviestudios',
	219	},
	220	'playlist_mincount': 80,
	221	}]
	222
223	def _real_extract(self, url):
224	section = self._match_id(url)
225	section_data = self._download_json(
226	'http://trailers.apple.com/trailers/home/feeds/%s.json' % self._SECTIONS[section]['feed_path'],
227	section)
228	entries = [
229	self.url_result('http://trailers.apple.com' + e['location'])
230	for e in section_data]
231	return self.playlist_result(entries, section, self._SECTIONS[section]['title'])