]>
Commit | Line | Data |
---|---|---|
93f94209 | 1 | # coding: utf-8 |
22e7f1a6 PH |
2 | from __future__ import unicode_literals |
3 | ||
9779b63b | 4 | import re |
9779b63b JMF |
5 | |
6 | from .common import InfoExtractor | |
a1a530b0 | 7 | from ..utils import ( |
f86d543e | 8 | ExtractorError, |
5890eef6 S |
9 | determine_ext, |
10 | int_or_none, | |
27aede90 | 11 | unified_strdate, |
a1a530b0 PH |
12 | US_RATINGS, |
13 | ) | |
9779b63b JMF |
14 | |
15 | ||
16 | class PBSIE(InfoExtractor): | |
22e7f1a6 PH |
17 | _VALID_URL = r'''(?x)https?:// |
18 | (?: | |
27aede90 PH |
19 | # Direct video URL |
20 | video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? | | |
21 | # Article with embedded player (or direct video) | |
22 | (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) | | |
22e7f1a6 | 23 | # Player |
773c0b4b | 24 | video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/ |
22e7f1a6 PH |
25 | ) |
26 | ''' | |
9779b63b | 27 | |
4d9bd478 S |
28 | _TESTS = [ |
29 | { | |
30 | 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/', | |
31 | 'md5': 'ce1888486f0908d555a8093cac9a7362', | |
32 | 'info_dict': { | |
33 | 'id': '2365006249', | |
34 | 'ext': 'mp4', | |
a5dd9a0c | 35 | 'title': 'Constitution USA with Peter Sagal - A More Perfect Union', |
4d9bd478 S |
36 | 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a', |
37 | 'duration': 3190, | |
38 | }, | |
5b61070c YCH |
39 | 'params': { |
40 | 'skip_download': True, # requires ffmpeg | |
41 | }, | |
9779b63b | 42 | }, |
cd6b4836 S |
43 | { |
44 | 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/', | |
45 | 'md5': '143c98aa54a346738a3d78f54c925321', | |
46 | 'info_dict': { | |
47 | 'id': '2365297690', | |
48 | 'ext': 'mp4', | |
a5dd9a0c | 49 | 'title': 'FRONTLINE - Losing Iraq', |
cd6b4836 S |
50 | 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e', |
51 | 'duration': 5050, | |
52 | }, | |
5b61070c YCH |
53 | 'params': { |
54 | 'skip_download': True, # requires ffmpeg | |
55 | } | |
cd6b4836 | 56 | }, |
4d9bd478 S |
57 | { |
58 | 'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/', | |
59 | 'md5': 'b19856d7f5351b17a5ab1dc6a64be633', | |
60 | 'info_dict': { | |
61 | 'id': '2201174722', | |
62 | 'ext': 'mp4', | |
a5dd9a0c | 63 | 'title': 'PBS NewsHour - Cyber Schools Gain Popularity, but Quality Questions Persist', |
4d9bd478 S |
64 | 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28', |
65 | 'duration': 801, | |
66 | }, | |
67 | }, | |
756b046f PH |
68 | { |
69 | 'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/', | |
70 | 'md5': 'c62859342be2a0358d6c9eb306595978', | |
71 | 'info_dict': { | |
72 | 'id': '2365297708', | |
73 | 'ext': 'mp4', | |
74 | 'description': 'md5:68d87ef760660eb564455eb30ca464fe', | |
a5dd9a0c | 75 | 'title': 'Great Performances - Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full', |
756b046f PH |
76 | 'duration': 6559, |
77 | 'thumbnail': 're:^https?://.*\.jpg$', | |
5b61070c YCH |
78 | }, |
79 | 'params': { | |
80 | 'skip_download': True, # requires ffmpeg | |
81 | }, | |
27aede90 PH |
82 | }, |
83 | { | |
84 | 'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html', | |
85 | 'md5': '908f3e5473a693b266b84e25e1cf9703', | |
86 | 'info_dict': { | |
87 | 'id': '2365160389', | |
88 | 'display_id': 'killer-typhoon', | |
89 | 'ext': 'mp4', | |
90 | 'description': 'md5:c741d14e979fc53228c575894094f157', | |
a5dd9a0c | 91 | 'title': 'NOVA - Killer Typhoon', |
27aede90 PH |
92 | 'duration': 3172, |
93 | 'thumbnail': 're:^https?://.*\.jpg$', | |
94 | 'upload_date': '20140122', | |
5b61070c YCH |
95 | }, |
96 | 'params': { | |
97 | 'skip_download': True, # requires ffmpeg | |
98 | }, | |
9bbec552 S |
99 | }, |
100 | { | |
101 | 'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/', | |
102 | 'info_dict': { | |
103 | 'id': 'united-states-of-secrets', | |
104 | }, | |
105 | 'playlist_count': 2, | |
dbe1a935 YCH |
106 | }, |
107 | { | |
108 | 'url': 'http://www.pbs.org/wgbh/americanexperience/films/death/player/', | |
109 | 'info_dict': { | |
110 | 'id': '2280706814', | |
111 | 'display_id': 'player', | |
112 | 'ext': 'mp4', | |
a5dd9a0c | 113 | 'title': 'American Experience - Death and the Civil War', |
dbe1a935 YCH |
114 | 'description': 'American Experience, TV’s most-watched history series, brings to life the compelling stories from our past that inform our understanding of the world today.', |
115 | 'duration': 6705, | |
116 | 'thumbnail': 're:^https?://.*\.jpg$', | |
117 | }, | |
118 | 'params': { | |
119 | 'skip_download': True, # requires ffmpeg | |
5b61070c | 120 | }, |
a5dd9a0c | 121 | }, |
122 | { | |
123 | 'url': 'http://video.pbs.org/video/2365367186/', | |
124 | 'info_dict': { | |
125 | 'id': '2365367186', | |
126 | 'display_id': '2365367186', | |
127 | 'ext': 'mp4', | |
128 | 'title': 'To Catch A Comet - Full Episode', | |
129 | 'description': 'On November 12, 2014, billions of kilometers from Earth, spacecraft orbiter Rosetta and lander Philae did what no other had dared to attempt \u2014 land on the volatile surface of a comet as it zooms around the sun at 67,000 km/hr. The European Space Agency hopes this mission can help peer into our past and unlock secrets of our origins.', | |
130 | 'duration': 3342, | |
131 | 'thumbnail': 're:^https?://.*\.jpg$', | |
132 | }, | |
133 | 'params': { | |
134 | 'skip_download': True, # requires ffmpeg | |
135 | }, | |
756b046f | 136 | } |
4d9bd478 | 137 | ] |
9779b63b | 138 | |
27aede90 | 139 | def _extract_webpage(self, url): |
9779b63b | 140 | mobj = re.match(self._VALID_URL, url) |
22e7f1a6 PH |
141 | |
142 | presumptive_id = mobj.group('presumptive_id') | |
143 | display_id = presumptive_id | |
144 | if presumptive_id: | |
145 | webpage = self._download_webpage(url, display_id) | |
8d7d9d34 | 146 | |
27aede90 PH |
147 | upload_date = unified_strdate(self._search_regex( |
148 | r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"', | |
149 | webpage, 'upload date', default=None)) | |
150 | ||
9bbec552 S |
151 | # tabbed frontline videos |
152 | tabbed_videos = re.findall( | |
153 | r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage) | |
154 | if tabbed_videos: | |
155 | return tabbed_videos, presumptive_id, upload_date | |
156 | ||
4d9bd478 S |
157 | MEDIA_ID_REGEXES = [ |
158 | r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed | |
159 | r'class="coveplayerid">([^<]+)<', # coveplayer | |
27aede90 | 160 | r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>', # jwplayer |
4d9bd478 S |
161 | ] |
162 | ||
8d7d9d34 | 163 | media_id = self._search_regex( |
4d9bd478 | 164 | MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None) |
8d7d9d34 | 165 | if media_id: |
27aede90 | 166 | return media_id, presumptive_id, upload_date |
8d7d9d34 | 167 | |
22e7f1a6 | 168 | url = self._search_regex( |
dbe1a935 | 169 | r'<iframe\s+[^>]*\s+src=["\']([^\'"]+partnerplayer[^\'"]+)["\']', |
22e7f1a6 PH |
170 | webpage, 'player URL') |
171 | mobj = re.match(self._VALID_URL, url) | |
172 | ||
173 | player_id = mobj.group('player_id') | |
174 | if not display_id: | |
175 | display_id = player_id | |
176 | if player_id: | |
177 | player_page = self._download_webpage( | |
178 | url, display_id, note='Downloading player page', | |
179 | errnote='Could not download player page') | |
180 | video_id = self._search_regex( | |
181 | r'<div\s+id="video_([0-9]+)"', player_page, 'video ID') | |
182 | else: | |
183 | video_id = mobj.group('id') | |
184 | display_id = video_id | |
185 | ||
27aede90 | 186 | return video_id, display_id, None |
8d7d9d34 S |
187 | |
188 | def _real_extract(self, url): | |
27aede90 | 189 | video_id, display_id, upload_date = self._extract_webpage(url) |
8d7d9d34 | 190 | |
9bbec552 S |
191 | if isinstance(video_id, list): |
192 | entries = [self.url_result( | |
193 | 'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id) | |
194 | for vid_id in video_id] | |
195 | return self.playlist_result(entries, display_id) | |
196 | ||
5890eef6 S |
197 | info = self._download_json( |
198 | 'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id, | |
199 | display_id) | |
200 | ||
201 | formats = [] | |
202 | for encoding_name in ('recommended_encoding', 'alternate_encoding'): | |
203 | redirect = info.get(encoding_name) | |
204 | if not redirect: | |
205 | continue | |
206 | redirect_url = redirect.get('url') | |
207 | if not redirect_url: | |
208 | continue | |
209 | ||
210 | redirect_info = self._download_json( | |
211 | redirect_url + '?format=json', display_id, | |
212 | 'Downloading %s video url info' % encoding_name) | |
213 | ||
214 | if redirect_info['status'] == 'error': | |
215 | if redirect_info['http_code'] == 403: | |
216 | message = ( | |
217 | 'The video is not available in your region due to ' | |
218 | 'right restrictions') | |
219 | else: | |
220 | message = redirect_info['message'] | |
221 | raise ExtractorError(message, expected=True) | |
222 | ||
223 | format_url = redirect_info.get('url') | |
224 | if not format_url: | |
225 | continue | |
226 | ||
227 | if determine_ext(format_url) == 'm3u8': | |
228 | formats.extend(self._extract_m3u8_formats( | |
229 | format_url, display_id, 'mp4', preference=1, m3u8_id='hls')) | |
f86d543e | 230 | else: |
5890eef6 S |
231 | formats.append({ |
232 | 'url': format_url, | |
e8b9ab89 | 233 | 'format_id': redirect.get('eeid'), |
5890eef6 S |
234 | }) |
235 | self._sort_formats(formats) | |
f86d543e | 236 | |
a1a530b0 PH |
237 | rating_str = info.get('rating') |
238 | if rating_str is not None: | |
239 | rating_str = rating_str.rpartition('-')[2] | |
240 | age_limit = US_RATINGS.get(rating_str) | |
241 | ||
06a12933 S |
242 | subtitles = {} |
243 | closed_captions_url = info.get('closed_captions_url') | |
244 | if closed_captions_url: | |
245 | subtitles['en'] = [{ | |
246 | 'ext': 'ttml', | |
247 | 'url': closed_captions_url, | |
248 | }] | |
249 | ||
ce1bafdc S |
250 | # info['title'] is often incomplete (e.g. 'Full Episode', 'Episode 5', etc) |
251 | # Try turning it to 'program - title' naming scheme if possible | |
0eacd2aa | 252 | alt_title = info.get('program', {}).get('title') |
253 | if alt_title: | |
c7620992 | 254 | info['title'] = alt_title + ' - ' + re.sub(r'^' + alt_title + '[\s\-:]+', '', info['title']) |
a5dd9a0c | 255 | |
22e7f1a6 PH |
256 | return { |
257 | 'id': video_id, | |
27aede90 | 258 | 'display_id': display_id, |
22e7f1a6 | 259 | 'title': info['title'], |
22e7f1a6 PH |
260 | 'description': info['program'].get('description'), |
261 | 'thumbnail': info.get('image_url'), | |
5890eef6 | 262 | 'duration': int_or_none(info.get('duration')), |
a1a530b0 | 263 | 'age_limit': age_limit, |
27aede90 | 264 | 'upload_date': upload_date, |
5890eef6 | 265 | 'formats': formats, |
06a12933 | 266 | 'subtitles': subtitles, |
22e7f1a6 | 267 | } |