2 from __future__
import unicode_literals
8 from .common
import InfoExtractor
9 from .subtitles
import SubtitlesInfoExtractor
12 compat_urllib_request
,
14 get_element_by_attribute
,
22 class VimeoIE(SubtitlesInfoExtractor
):
23 """Information extractor for vimeo.com."""
25 # _VALID_URL matches Vimeo URLs
27 (?P<proto>(?:https?:)?//)?
28 (?:(?:www|(?P<player>player))\.)?
29 vimeo(?P<pro>pro)?\.com/
31 (?:(?:play_redirect_hls|moogaloop\.swf)\?clip_id=)?
34 /?(?:[?&].*)?(?:[#].*)?$'''
35 _NETRC_MACHINE
= 'vimeo'
39 'url': 'http://vimeo.com/56015672#at=0',
40 'md5': '8879b6cc097e987f02484baf890129e5',
44 "upload_date": "20121220",
45 "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
46 "uploader_id": "user7108434",
47 "uploader": "Filippo Valsorda",
48 "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
52 'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
53 'file': '68093876.mp4',
54 'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
55 'note': 'Vimeo Pro video (#1197)',
57 'uploader_id': 'openstreetmapus',
58 'uploader': 'OpenStreetMap US',
59 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
63 'url': 'http://player.vimeo.com/video/54469442',
64 'file': '54469442.mp4',
65 'md5': '619b811a4417aa4abe78dc653becf511',
66 'note': 'Videos that embed the url in the player page',
68 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software',
69 'uploader': 'The BLN & Business of Software',
70 'uploader_id': 'theblnbusinessofsoftware',
74 'url': 'http://vimeo.com/68375962',
75 'file': '68375962.mp4',
76 'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
77 'note': 'Video protected with password',
79 'title': 'youtube-dl password protected test video',
80 'upload_date': '20130614',
81 'uploader_id': 'user18948128',
82 'uploader': 'Jaime Marquínez Ferrándiz',
85 'videopassword': 'youtube-dl',
89 'url': 'http://vimeo.com/76979871',
90 'md5': '3363dd6ffebe3784d56f4132317fd446',
91 'note': 'Video with subtitles',
95 'title': 'The New Vimeo Player (You Know, For Videos)',
96 'description': 'md5:2ec900bf97c3f389378a96aee11260ea',
97 'upload_date': '20131015',
98 'uploader_id': 'staff',
99 'uploader': 'Vimeo Staff',
105 (username
, password
) = self
._get
_login
_info
()
109 login_url
= 'https://vimeo.com/log_in'
110 webpage
= self
._download
_webpage
(login_url
, None, False)
111 token
= self
._search
_regex
(r
'xsrft: \'(.*?
)\'', webpage, 'login token
')
112 data = compat_urllib_parse.urlencode({'email
': username,
113 'password
': password,
118 login_request = compat_urllib_request.Request(login_url, data)
119 login_request.add_header('Content
-Type
', 'application
/x
-www
-form
-urlencoded
')
120 login_request.add_header('Cookie
', 'xsrft
=%s' % token)
121 self._download_webpage(login_request, None, False, 'Wrong login info
')
123 def _verify_video_password(self, url, video_id, webpage):
124 password = self._downloader.params.get('videopassword
', None)
126 raise ExtractorError('This video
is protected by a password
, use the
--video
-password option
')
127 token = self._search_regex(r'xsrft
: \'(.*?
)\'', webpage, 'login token
')
128 data = compat_urllib_parse.urlencode({'password
': password,
130 # I didn't manage to use the password
with https
131 if url
.startswith('https'):
132 pass_url
= url
.replace('https','http')
135 password_request
= compat_urllib_request
.Request(pass_url
+'/password', data
)
136 password_request
.add_header('Content-Type', 'application/x-www-form-urlencoded')
137 password_request
.add_header('Cookie', 'xsrft=%s' % token
)
138 self
._download
_webpage
(password_request
, video_id
,
139 'Verifying the password',
142 def _verify_player_video_password(self
, url
, video_id
):
143 password
= self
._downloader
.params
.get('videopassword', None)
145 raise ExtractorError('This video is protected by a password, use the --video-password option')
146 data
= compat_urllib_parse
.urlencode({'password': password}
)
147 pass_url
= url
+ '/check-password'
148 password_request
= compat_urllib_request
.Request(pass_url
, data
)
149 password_request
.add_header('Content-Type', 'application/x-www-form-urlencoded')
150 return self
._download
_json
(
151 password_request
, video_id
,
152 'Verifying the password',
155 def _real_initialize(self
):
158 def _real_extract(self
, url
):
159 url
, data
= unsmuggle_url(url
)
160 headers
= std_headers
162 headers
= headers
.copy()
165 # Extract ID from URL
166 mobj
= re
.match(self
._VALID
_URL
, url
)
167 video_id
= mobj
.group('id')
168 if mobj
.group('pro') or mobj
.group('player'):
169 url
= 'http://player.vimeo.com/video/' + video_id
171 url
= 'https://vimeo.com/' + video_id
173 # Retrieve video webpage to extract further information
174 request
= compat_urllib_request
.Request(url
, None, headers
)
175 webpage
= self
._download
_webpage
(request
, video_id
)
177 # Now we begin extracting as much information as we can from what we
178 # retrieved. First we extract the information common to all extractors,
179 # and latter we extract those that are Vimeo specific.
180 self
.report_extraction(video_id
)
182 # Extract the config JSON
185 config_url
= self
._html
_search
_regex
(
186 r
' data-config-url="(.+?)"', webpage
, 'config URL')
187 config_json
= self
._download
_webpage
(config_url
, video_id
)
188 config
= json
.loads(config_json
)
189 except RegexNotFoundError
:
190 # For pro videos or player.vimeo.com urls
191 # We try to find out to which variable is assigned the config dic
192 m_variable_name
= re
.search('(\w)\.video\.id', webpage
)
193 if m_variable_name
is not None:
194 config_re
= r
'%s=({.+?});' % re
.escape(m_variable_name
.group(1))
196 config_re
= [r
' = {config:({.+?}),assets:', r
'(?:[abc])=({.+?});']
197 config
= self
._search
_regex
(config_re
, webpage
, 'info section',
199 config
= json
.loads(config
)
200 except Exception as e
:
201 if re
.search('The creator of this video has not given you permission to embed it on this domain.', webpage
):
202 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
204 if re
.search('<form[^>]+?id="pw_form"', webpage
) is not None:
205 self
._verify
_video
_password
(url
, video_id
, webpage
)
206 return self
._real
_extract
(url
)
208 raise ExtractorError('Unable to extract info section',
211 if config
.get('view') == 4:
212 config
= self
._verify
_player
_video
_password
(url
, video_id
)
215 video_title
= config
["video"]["title"]
217 # Extract uploader and uploader_id
218 video_uploader
= config
["video"]["owner"]["name"]
219 video_uploader_id
= config
["video"]["owner"]["url"].split('/')[-1] if config
["video"]["owner"]["url"] else None
221 # Extract video thumbnail
222 video_thumbnail
= config
["video"].get("thumbnail")
223 if video_thumbnail
is None and config
["video"].get("thumbs"):
224 _
, video_thumbnail
= sorted((int(width
), t_url
) for (width
, t_url
) in config
["video"]["thumbs"].items())[-1]
226 # Extract video description
227 video_description
= None
229 video_description
= get_element_by_attribute("itemprop", "description", webpage
)
230 if video_description
: video_description
= clean_html(video_description
)
231 except AssertionError as err
:
232 # On some pages like (http://player.vimeo.com/video/54469442) the
233 # html tags are not closed, python 2.6 cannot handle it
234 if err
.args
[0] == 'we should not get here!':
239 # Extract upload date
240 video_upload_date
= None
241 mobj
= re
.search(r
'<meta itemprop="dateCreated" content="(\d{4})-(\d{2})-(\d{2})T', webpage
)
243 video_upload_date
= mobj
.group(1) + mobj
.group(2) + mobj
.group(3)
246 view_count
= int(self
._search
_regex
(r
'UserPlays:(\d+)', webpage
, 'view count'))
247 like_count
= int(self
._search
_regex
(r
'UserLikes:(\d+)', webpage
, 'like count'))
248 comment_count
= int(self
._search
_regex
(r
'UserComments:(\d+)', webpage
, 'comment count'))
249 except RegexNotFoundError
:
250 # This info is only available in vimeo.com/{id} urls
255 # Vimeo specific: extract request signature and timestamp
256 sig
= config
['request']['signature']
257 timestamp
= config
['request']['timestamp']
259 # Vimeo specific: extract video codec and quality information
260 # First consider quality, then codecs, then take everything
261 codecs
= [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
262 files
= {'hd': [], 'sd': [], 'other': []}
263 config_files
= config
["video"].get("files") or config
["request"].get("files")
264 for codec_name
, codec_extension
in codecs
:
265 for quality
in config_files
.get(codec_name
, []):
266 format_id
= '-'.join((codec_name
, quality
)).lower()
267 key
= quality
if quality
in files
else 'other'
269 if isinstance(config_files
[codec_name
], dict):
270 file_info
= config_files
[codec_name
][quality
]
271 video_url
= file_info
.get('url')
274 if video_url
is None:
275 video_url
= "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
276 %(video_id
, sig
, timestamp
, quality
, codec_name
.upper())
279 'ext': codec_extension
,
281 'format_id': format_id
,
282 'width': file_info
.get('width'),
283 'height': file_info
.get('height'),
286 for key
in ('other', 'sd', 'hd'):
287 formats
+= files
[key
]
288 if len(formats
) == 0:
289 raise ExtractorError('No known codec found')
292 text_tracks
= config
['request'].get('text_tracks')
294 for tt
in text_tracks
:
295 subtitles
[tt
['lang']] = 'http://vimeo.com' + tt
['url']
297 video_subtitles
= self
.extract_subtitles(video_id
, subtitles
)
298 if self
._downloader
.params
.get('listsubtitles', False):
299 self
._list
_available
_subtitles
(video_id
, subtitles
)
304 'uploader': video_uploader
,
305 'uploader_id': video_uploader_id
,
306 'upload_date': video_upload_date
,
307 'title': video_title
,
308 'thumbnail': video_thumbnail
,
309 'description': video_description
,
312 'view_count': view_count
,
313 'like_count': like_count
,
314 'comment_count': comment_count
,
315 'subtitles': video_subtitles
,
319 class VimeoChannelIE(InfoExtractor
):
320 IE_NAME
= 'vimeo:channel'
321 _VALID_URL
= r
'(?:https?://)?vimeo\.com/channels/(?P<id>[^/]+)'
322 _MORE_PAGES_INDICATOR
= r
'<a.+?rel="next"'
323 _TITLE_RE
= r
'<link rel="alternate"[^>]+?title="(.*?)"'
325 def _page_url(self
, base_url
, pagenum
):
326 return '%s/videos/page:%d/' % (base_url
, pagenum
)
328 def _extract_list_title(self
, webpage
):
329 return self
._html
_search
_regex
(self
._TITLE
_RE
, webpage
, 'list title')
331 def _extract_videos(self
, list_id
, base_url
):
333 for pagenum
in itertools
.count(1):
334 webpage
= self
._download
_webpage
(
335 self
._page
_url
(base_url
, pagenum
) ,list_id
,
336 'Downloading page %s' % pagenum
)
337 video_ids
.extend(re
.findall(r
'id="clip_(\d+?)"', webpage
))
338 if re
.search(self
._MORE
_PAGES
_INDICATOR
, webpage
, re
.DOTALL
) is None:
341 entries
= [self
.url_result('http://vimeo.com/%s' % video_id
, 'Vimeo')
342 for video_id
in video_ids
]
343 return {'_type': 'playlist',
345 'title': self
._extract
_list
_title
(webpage
),
349 def _real_extract(self
, url
):
350 mobj
= re
.match(self
._VALID
_URL
, url
)
351 channel_id
= mobj
.group('id')
352 return self
._extract
_videos
(channel_id
, 'http://vimeo.com/channels/%s' % channel_id
)
355 class VimeoUserIE(VimeoChannelIE
):
356 IE_NAME
= 'vimeo:user'
357 _VALID_URL
= r
'(?:https?://)?vimeo\.com/(?P<name>[^/]+)(?:/videos|[#?]|$)'
358 _TITLE_RE
= r
'<a[^>]+?class="user">([^<>]+?)</a>'
361 def suitable(cls
, url
):
362 if VimeoChannelIE
.suitable(url
) or VimeoIE
.suitable(url
) or VimeoAlbumIE
.suitable(url
) or VimeoGroupsIE
.suitable(url
):
364 return super(VimeoUserIE
, cls
).suitable(url
)
366 def _real_extract(self
, url
):
367 mobj
= re
.match(self
._VALID
_URL
, url
)
368 name
= mobj
.group('name')
369 return self
._extract
_videos
(name
, 'http://vimeo.com/%s' % name
)
372 class VimeoAlbumIE(VimeoChannelIE
):
373 IE_NAME
= 'vimeo:album'
374 _VALID_URL
= r
'(?:https?://)?vimeo\.com/album/(?P<id>\d+)'
375 _TITLE_RE
= r
'<header id="page_header">\n\s*<h1>(.*?)</h1>'
377 def _page_url(self
, base_url
, pagenum
):
378 return '%s/page:%d/' % (base_url
, pagenum
)
380 def _real_extract(self
, url
):
381 mobj
= re
.match(self
._VALID
_URL
, url
)
382 album_id
= mobj
.group('id')
383 return self
._extract
_videos
(album_id
, 'http://vimeo.com/album/%s' % album_id
)
386 class VimeoGroupsIE(VimeoAlbumIE
):
387 IE_NAME
= 'vimeo:group'
388 _VALID_URL
= r
'(?:https?://)?vimeo\.com/groups/(?P<name>[^/]+)'
390 def _extract_list_title(self
, webpage
):
391 return self
._og
_search
_title
(webpage
)
393 def _real_extract(self
, url
):
394 mobj
= re
.match(self
._VALID
_URL
, url
)
395 name
= mobj
.group('name')
396 return self
._extract
_videos
(name
, 'http://vimeo.com/groups/%s' % name
)
399 class VimeoReviewIE(InfoExtractor
):
400 IE_NAME
= 'vimeo:review'
401 IE_DESC
= 'Review pages on vimeo'
402 _VALID_URL
= r
'(?:https?://)?vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
404 'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
405 'file': '75524534.mp4',
406 'md5': 'c507a72f780cacc12b2248bb4006d253',
408 'title': "DICK HARDWICK 'Comedian'",
409 'uploader': 'Richard Hardwick',
413 def _real_extract(self
, url
):
414 mobj
= re
.match(self
._VALID
_URL
, url
)
415 video_id
= mobj
.group('id')
416 player_url
= 'https://player.vimeo.com/player/' + video_id
417 return self
.url_result(player_url
, 'Vimeo', video_id
)