[yt-dlp.git] / youtube_dl / extractor / appletrailers.py

import re
import xml.etree.ElementTree

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    ExtractorError,
)


class AppleTrailersIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?trailers.apple.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
    _TEST = {
        u"url": u"http://trailers.apple.com/trailers/wb/manofsteel/",
        u"playlist": [
            {
                u"file": u"manofsteel-trailer4.mov",
                u"md5": u"11874af099d480cc09e103b189805d5f",
                u"info_dict": {
                    u"duration": 111,
                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_11624.jpg",
                    u"title": u"Trailer 4",
                    u"upload_date": u"20130523",
                    u"uploader_id": u"wb",
                },
            },
            {
                u"file": u"manofsteel-trailer3.mov",
                u"md5": u"07a0a262aae5afe68120eed61137ab34",
                u"info_dict": {
                    u"duration": 182,
                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_10793.jpg",
                    u"title": u"Trailer 3",
                    u"upload_date": u"20130417",
                    u"uploader_id": u"wb",
                },
            },
            {
                u"file": u"manofsteel-trailer.mov",
                u"md5": u"e401fde0813008e3307e54b6f384cff1",
                u"info_dict": {
                    u"duration": 148,
                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_8703.jpg",
                    u"title": u"Trailer",
                    u"upload_date": u"20121212",
                    u"uploader_id": u"wb",
                },
            },
            {
                u"file": u"manofsteel-teaser.mov",
                u"md5": u"76b392f2ae9e7c98b22913c10a639c97",
                u"info_dict": {
                    u"duration": 93,
                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_6899.jpg",
                    u"title": u"Teaser",
                    u"upload_date": u"20120721",
                    u"uploader_id": u"wb",
                },
            }
        ]
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

        playlist_url = url.partition(u'?')[0] + u'/includes/playlists/web.inc'
        playlist_snippet = self._download_webpage(playlist_url, movie)
        playlist_cleaned = re.sub(r'(?s)<script>.*?</script>', u'', playlist_snippet)
        playlist_html = u'<html>' + playlist_cleaned + u'</html>'

        size_cache = {}

        doc = xml.etree.ElementTree.fromstring(playlist_html)
        playlist = []
        for li in doc.findall('./div/ul/li'):
            title = li.find('.//h3').text
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']

            date_el = li.find('.//p')
            upload_date = None
            m = re.search(r':\s?(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<year>[0-9]{2})', date_el.text)
            if m:
                upload_date = u'20' + m.group('year') + m.group('month') + m.group('day')
            runtime_el = date_el.find('./br')
            m = re.search(r':\s?(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime_el.tail)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

            formats = []
            for formats_el in li.findall('.//li/a'):
                if formats_el.attrib['class'] != 'OverlayPanel':
                    continue
                target = formats_el.attrib['target']

                format_code = formats_el.text
                if 'Automatic' in format_code:
                    continue

                size_q = formats_el.attrib['href']
                size_id = size_q.rpartition('#videos-')[2]
                if size_id not in size_cache:
                    size_url = url + size_q
                    sizepage_html = self._download_webpage(
                        size_url, movie,
                        note=u'Downloading size info %s' % size_id,
                        errnote=u'Error while downloading size info %s' % size_id,
                    )
                    _doc = xml.etree.ElementTree.fromstring(sizepage_html)
                    size_cache[size_id] = _doc

                sizepage_doc = size_cache[size_id]
                links = sizepage_doc.findall('.//{http://www.w3.org/1999/xhtml}ul/{http://www.w3.org/1999/xhtml}li/{http://www.w3.org/1999/xhtml}a')
                for vid_a in links:
                    href = vid_a.get('href')
                    if not href.endswith(target):
                        continue
                    detail_q = href.partition('#')[0]
                    detail_url = url + '/' + detail_q

                    m = re.match(r'includes/(?P<detail_id>[^/]+)/', detail_q)
                    detail_id = m.group('detail_id')

                    detail_html = self._download_webpage(
                        detail_url, movie,
                        note=u'Downloading detail %s %s' % (detail_id, size_id),
                        errnote=u'Error while downloading detail %s %s' % (detail_id, size_id)
                    )
                    detail_doc = xml.etree.ElementTree.fromstring(detail_html)
                    movie_link_el = detail_doc.find('.//{http://www.w3.org/1999/xhtml}a')
                    assert movie_link_el.get('class') == 'movieLink'
                    movie_link = movie_link_el.get('href').partition('?')[0].replace('_', '_h')
                    ext = determine_ext(movie_link)
                    assert ext == 'mov'

                    formats.append({
                        'format': format_code,
                        'ext': ext,
                        'url': movie_link,
                    })

            info = {
                '_type': 'video',
                'id': video_id,
                'title': title,
                'formats': formats,
                'title': title,
                'duration': duration,
                'thumbnail': thumbnail,
                'upload_date': upload_date,
                'uploader_id': uploader_id,
                'user_agent': 'QuickTime compatible (youtube-dl)',
            }
            # TODO: Remove when #980 has been merged
            info['url'] = formats[-1]['url']
            info['ext'] = formats[-1]['ext']

            playlist.append(info)

        return {
            '_type': 'playlist',
            'id': movie,
            'entries': playlist,
        }
Commit	Line	Data
44586389 PH	1	import re
	2	import xml.etree.ElementTree
	3
	4	from .common import InfoExtractor
	5	from ..utils import (
	6	determine_ext,
	7	ExtractorError,
	8	)
	9
	10
	11	class AppleTrailersIE(InfoExtractor):
	12	_VALID_URL = r'https?://(?:www\.)?trailers.apple.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
	13	_TEST = {
	14	u"url": u"http://trailers.apple.com/trailers/wb/manofsteel/",
	15	u"playlist": [
	16	{
	17	u"file": u"manofsteel-trailer4.mov",
	18	u"md5": u"11874af099d480cc09e103b189805d5f",
	19	u"info_dict": {
	20	u"duration": 111,
	21	u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_11624.jpg",
	22	u"title": u"Trailer 4",
	23	u"upload_date": u"20130523",
	24	u"uploader_id": u"wb",
	25	},
	26	},
	27	{
	28	u"file": u"manofsteel-trailer3.mov",
	29	u"md5": u"07a0a262aae5afe68120eed61137ab34",
	30	u"info_dict": {
	31	u"duration": 182,
	32	u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_10793.jpg",
	33	u"title": u"Trailer 3",
	34	u"upload_date": u"20130417",
	35	u"uploader_id": u"wb",
	36	},
	37	},
	38	{
	39	u"file": u"manofsteel-trailer.mov",
	40	u"md5": u"e401fde0813008e3307e54b6f384cff1",
	41	u"info_dict": {
	42	u"duration": 148,
	43	u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_8703.jpg",
	44	u"title": u"Trailer",
	45	u"upload_date": u"20121212",
	46	u"uploader_id": u"wb",
	47	},
	48	},
	49	{
	50	u"file": u"manofsteel-teaser.mov",
	51	u"md5": u"76b392f2ae9e7c98b22913c10a639c97",
	52	u"info_dict": {
	53	u"duration": 93,
	54	u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_6899.jpg",
	55	u"title": u"Teaser",
	56	u"upload_date": u"20120721",
	57	u"uploader_id": u"wb",
	58	},
	59	}
	60	]
	61	}
	62
	63	def _real_extract(self, url):
	64	mobj = re.match(self._VALID_URL, url)
65	movie = mobj.group('movie')
66	uploader_id = mobj.group('company')
67
68	playlist_url = url.partition(u'?')[0] + u'/includes/playlists/web.inc'
69	playlist_snippet = self._download_webpage(playlist_url, movie)
70	playlist_cleaned = re.sub(r'(?s)<script>.*?</script>', u'', playlist_snippet)
71	playlist_html = u'<html>' + playlist_cleaned + u'</html>'
72
73	size_cache = {}
74
75	doc = xml.etree.ElementTree.fromstring(playlist_html)
76	playlist = []
77	for li in doc.findall('./div/ul/li'):
78	title = li.find('.//h3').text
79	video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
80	thumbnail = li.find('.//img').attrib['src']
81
82	date_el = li.find('.//p')
83	upload_date = None
84	m = re.search(r':\s?(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<year>[0-9]{2})', date_el.text)
85	if m:
86	upload_date = u'20' + m.group('year') + m.group('month') + m.group('day')
87	runtime_el = date_el.find('./br')
88	m = re.search(r':\s?(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime_el.tail)
89	duration = None
90	if m:
91	duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
92
93	formats = []
94	for formats_el in li.findall('.//li/a'):
95	if formats_el.attrib['class'] != 'OverlayPanel':
96	continue
97	target = formats_el.attrib['target']
98
99	format_code = formats_el.text
100	if 'Automatic' in format_code:
101	continue
102
103	size_q = formats_el.attrib['href']
104	size_id = size_q.rpartition('#videos-')[2]
105	if size_id not in size_cache:
106	size_url = url + size_q
107	sizepage_html = self._download_webpage(
108	size_url, movie,
109	note=u'Downloading size info %s' % size_id,
110	errnote=u'Error while downloading size info %s' % size_id,
111	)
112	_doc = xml.etree.ElementTree.fromstring(sizepage_html)
113	size_cache[size_id] = _doc
114
115	sizepage_doc = size_cache[size_id]
116	links = sizepage_doc.findall('.//{http://www.w3.org/1999/xhtml}ul/{http://www.w3.org/1999/xhtml}li/{http://www.w3.org/1999/xhtml}a')
117	for vid_a in links:
118	href = vid_a.get('href')
119	if not href.endswith(target):
120	continue
121	detail_q = href.partition('#')[0]
122	detail_url = url + '/' + detail_q
123
124	m = re.match(r'includes/(?P<detail_id>[^/]+)/', detail_q)
125	detail_id = m.group('detail_id')
126
127	detail_html = self._download_webpage(
128	detail_url, movie,
129	note=u'Downloading detail %s %s' % (detail_id, size_id),
130	errnote=u'Error while downloading detail %s %s' % (detail_id, size_id)
131	)
132	detail_doc = xml.etree.ElementTree.fromstring(detail_html)
133	movie_link_el = detail_doc.find('.//{http://www.w3.org/1999/xhtml}a')
134	assert movie_link_el.get('class') == 'movieLink'
135	movie_link = movie_link_el.get('href').partition('?')[0].replace('_', '_h')
136	ext = determine_ext(movie_link)
137	assert ext == 'mov'
138
139	formats.append({
140	'format': format_code,
141	'ext': ext,
142	'url': movie_link,
143	})
144
145	info = {
146	'_type': 'video',
147	'id': video_id,
148	'title': title,
149	'formats': formats,
150	'title': title,
151	'duration': duration,
152	'thumbnail': thumbnail,
153	'upload_date': upload_date,
154	'uploader_id': uploader_id,
155	'user_agent': 'QuickTime compatible (youtube-dl)',
156	}
157	# TODO: Remove when #980 has been merged
158	info['url'] = formats[-1]['url']
159	info['ext'] = formats[-1]['ext']
160
161	playlist.append(info)
162
163	return {
164	'_type': 'playlist',
165	'id': movie,
166	'entries': playlist,
167	}