[yt-dlp.git] / youtube_dl / extractor / appletrailers.py

from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
)


class AppleTrailersIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
    _TESTS = [{
        "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
        'info_dict': {
            'id': 'manofsteel',
        },
        "playlist": [
            {
                "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
                "info_dict": {
                    "id": "manofsteel-trailer4",
                    "ext": "mov",
                    "duration": 111,
                    "title": "Trailer 4",
                    "upload_date": "20130523",
                    "uploader_id": "wb",
                },
            },
            {
                "md5": "b8017b7131b721fb4e8d6f49e1df908c",
                "info_dict": {
                    "id": "manofsteel-trailer3",
                    "ext": "mov",
                    "duration": 182,
                    "title": "Trailer 3",
                    "upload_date": "20130417",
                    "uploader_id": "wb",
                },
            },
            {
                "md5": "d0f1e1150989b9924679b441f3404d48",
                "info_dict": {
                    "id": "manofsteel-trailer",
                    "ext": "mov",
                    "duration": 148,
                    "title": "Trailer",
                    "upload_date": "20121212",
                    "uploader_id": "wb",
                },
            },
            {
                "md5": "5fe08795b943eb2e757fa95cb6def1cb",
                "info_dict": {
                    "id": "manofsteel-teaser",
                    "ext": "mov",
                    "duration": 93,
                    "title": "Teaser",
                    "upload_date": "20120721",
                    "uploader_id": "wb",
                },
            },
        ]
    }, {
        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
        'only_matching': True,
    }]

    _JSON_RE = r'iTunes.playURL\((.*?)\);'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

        playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')

        def fix_html(s):
            s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
            s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
            # The ' in the onClick attributes are not escaped, it couldn't be parsed
            # like: http://trailers.apple.com/trailers/wb/gravity/

            def _clean_json(m):
                return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
            s = re.sub(self._JSON_RE, _clean_json, s)
            s = '<html>%s</html>' % s
            return s
        doc = self._download_xml(playlist_url, movie, transform_source=fix_html)

        playlist = []
        for li in doc.findall('./div/ul/li'):
            on_click = li.find('.//a').attrib['onClick']
            trailer_info_json = self._search_regex(self._JSON_RE,
                                                   on_click, 'trailer info')
            trailer_info = json.loads(trailer_info_json)
            title = trailer_info['title']
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']
            upload_date = trailer_info['posted'].replace('-', '')

            runtime = trailer_info['runtime']
            m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

            first_url = trailer_info['url']
            trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
            settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
            settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

            formats = []
            for format in settings['metadata']['sizes']:
                # The src is a file pointing to the real video file
                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
                formats.append({
                    'url': format_url,
                    'format': format['type'],
                    'width': int_or_none(format['width']),
                    'height': int_or_none(format['height']),
                })

            self._sort_formats(formats)

            playlist.append({
                '_type': 'video',
                'id': video_id,
                'formats': formats,
                'title': title,
                'duration': duration,
                'thumbnail': thumbnail,
                'upload_date': upload_date,
                'uploader_id': uploader_id,
                'http_headers': {
                    'User-Agent': 'QuickTime compatible (youtube-dl)',
                },
            })

        return {
            '_type': 'playlist',
            'id': movie,
            'entries': playlist,
        }
Commit	Line	Data
3798eadc PH	1	from __future__ import unicode_literals
3798eadc PH	2
44586389	3	import re
84353056	4	import json
44586389 PH	5
44586389 PH	6	from .common import InfoExtractor
1cc79574	7	from ..compat import compat_urlparse
44586389	8	from ..utils import (
9572013d	9	int_or_none,
44586389 PH	10	)
	11
	12
	13	class AppleTrailersIE(InfoExtractor):
f311cfa2	14	_VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers\|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
35b79823	15	_TESTS = [{
3798eadc	16	"url": "http://trailers.apple.com/trailers/wb/manofsteel/",
11e611a7 PH	17	'info_dict': {
	18	'id': 'manofsteel',
	19	},
3798eadc	20	"playlist": [
44586389	21	{
3798eadc PH	22	"md5": "d97a8e575432dbcb81b7c3acb741f8a8",
3798eadc PH	23	"info_dict": {
28acf550 JMF	24	"id": "manofsteel-trailer4",
28acf550 JMF	25	"ext": "mov",
3798eadc PH	26	"duration": 111,
	27	"title": "Trailer 4",
	28	"upload_date": "20130523",
	29	"uploader_id": "wb",
44586389 PH	30	},
	31	},
	32	{
3798eadc PH	33	"md5": "b8017b7131b721fb4e8d6f49e1df908c",
3798eadc PH	34	"info_dict": {
28acf550 JMF	35	"id": "manofsteel-trailer3",
28acf550 JMF	36	"ext": "mov",
3798eadc PH	37	"duration": 182,
	38	"title": "Trailer 3",
	39	"upload_date": "20130417",
	40	"uploader_id": "wb",
44586389 PH	41	},
	42	},
	43	{
3798eadc PH	44	"md5": "d0f1e1150989b9924679b441f3404d48",
3798eadc PH	45	"info_dict": {
28acf550 JMF	46	"id": "manofsteel-trailer",
28acf550 JMF	47	"ext": "mov",
3798eadc PH	48	"duration": 148,
	49	"title": "Trailer",
	50	"upload_date": "20121212",
	51	"uploader_id": "wb",
44586389 PH	52	},
	53	},
	54	{
3798eadc PH	55	"md5": "5fe08795b943eb2e757fa95cb6def1cb",
3798eadc PH	56	"info_dict": {
28acf550 JMF	57	"id": "manofsteel-teaser",
28acf550 JMF	58	"ext": "mov",
3798eadc PH	59	"duration": 93,
	60	"title": "Teaser",
	61	"upload_date": "20120721",
	62	"uploader_id": "wb",
44586389	63	},
28acf550	64	},
44586389	65	]
35b79823 S	66	}, {
	67	'url': 'http://trailers.apple.com/ca/metropole/autrui/',
	68	'only_matching': True,
	69	}]
44586389	70
84353056 JMF	71	_JSON_RE = r'iTunes.playURL\((.*?)\);'
84353056 JMF	72
44586389 PH	73	def _real_extract(self, url):
	74	mobj = re.match(self._VALID_URL, url)
	75	movie = mobj.group('movie')
	76	uploader_id = mobj.group('company')
	77
28acf550	78	playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
5f6a1245	79
18258362	80	def fix_html(s):
28acf550	81	s = re.sub(r'(?s)<script[^<]?>.?</script>', '', s)
18258362 JMF	82	s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
	83	# The ' in the onClick attributes are not escaped, it couldn't be parsed
	84	# like: http://trailers.apple.com/trailers/wb/gravity/
5f6a1245	85
18258362	86	def _clean_json(m):
28acf550	87	return 'iTunes.playURL(%s);' % m.group(1).replace('\'', ''')
18258362	88	s = re.sub(self._JSON_RE, _clean_json, s)
e91cdcae	89	s = '<html>%s</html>' % s
18258362 JMF	90	return s
18258362 JMF	91	doc = self._download_xml(playlist_url, movie, transform_source=fix_html)
44586389	92
44586389 PH	93	playlist = []
44586389 PH	94	for li in doc.findall('./div/ul/li'):
84353056 JMF	95	on_click = li.find('.//a').attrib['onClick']
84353056 JMF	96	trailer_info_json = self._search_regex(self._JSON_RE,
9e1a5b84	97	on_click, 'trailer info')
84353056 JMF	98	trailer_info = json.loads(trailer_info_json)
84353056 JMF	99	title = trailer_info['title']
44586389 PH	100	video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
44586389 PH	101	thumbnail = li.find('.//img').attrib['src']
84353056	102	upload_date = trailer_info['posted'].replace('-', '')
44586389	103
84353056 JMF	104	runtime = trailer_info['runtime']
84353056 JMF	105	m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
44586389 PH	106	duration = None
	107	if m:
	108	duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
	109
84353056	110	first_url = trailer_info['url']
bb4aa62c	111	trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
84353056	112	settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
28acf550	113	settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')
44586389	114
84353056 JMF	115	formats = []
	116	for format in settings['metadata']['sizes']:
	117	# The src is a file pointing to the real video file
	118	format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
	119	formats.append({
	120	'url': format_url,
84353056	121	'format': format['type'],
9572013d PH	122	'width': int_or_none(format['width']),
9572013d PH	123	'height': int_or_none(format['height']),
84353056	124	})
7b8af563 PH	125
7b8af563 PH	126	self._sort_formats(formats)
44586389	127
fb7abb31	128	playlist.append({
44586389 PH	129	'_type': 'video',
44586389 PH	130	'id': video_id,
44586389 PH	131	'formats': formats,
	132	'title': title,
	133	'duration': duration,
	134	'thumbnail': thumbnail,
	135	'upload_date': upload_date,
	136	'uploader_id': uploader_id,
e1554a40 JMF	137	'http_headers': {
	138	'User-Agent': 'QuickTime compatible (youtube-dl)',
	139	},
fb7abb31	140	})
44586389 PH	141
	142	return {
	143	'_type': 'playlist',
	144	'id': movie,
	145	'entries': playlist,
	146	}