2 from __future__
import unicode_literals
8 from .common
import InfoExtractor
9 from .subtitles
import SubtitlesInfoExtractor
13 compat_urllib_request
,
15 get_element_by_attribute
,
24 class VimeoBaseInfoExtractor(InfoExtractor
):
25 _NETRC_MACHINE
= 'vimeo'
26 _LOGIN_REQUIRED
= False
29 (username
, password
) = self
._get
_login
_info
()
31 if self
._LOGIN
_REQUIRED
:
32 raise ExtractorError(u
'No login info available, needed for using %s.' % self
.IE_NAME
, expected
=True)
35 login_url
= 'https://vimeo.com/log_in'
36 webpage
= self
._download
_webpage
(login_url
, None, False)
37 token
= self
._search
_regex
(r
'xsrft: \'(.*?
)\'', webpage, 'login token
')
38 data = urlencode_postdata({
45 login_request = compat_urllib_request.Request(login_url, data)
46 login_request.add_header('Content
-Type
', 'application
/x
-www
-form
-urlencoded
')
47 login_request.add_header('Cookie
', 'xsrft
=%s' % token)
48 self._download_webpage(login_request, None, False, 'Wrong login info
')
51 class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
52 """Information extractor for vimeo.com."""
54 # _VALID_URL matches Vimeo URLs
56 (?P<proto>(?:https?:)?//)?
57 (?:(?:www|(?P<player>player))\.)?
58 vimeo(?P<pro>pro)?\.com/
60 (?:(?:play_redirect_hls|moogaloop\.swf)\?clip_id=)?
63 /?(?:[?&].*)?(?:[#].*)?$'''
67 'url
': 'http
://vimeo
.com
/56015672#at=0',
68 'md5': '8879b6cc097e987f02484baf890129e5',
72 "upload_date": "20121220",
73 "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
74 "uploader_id": "user7108434",
75 "uploader": "Filippo Valsorda",
76 "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
80 'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
81 'file': '68093876.mp4',
82 'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
83 'note': 'Vimeo Pro video (#1197)',
85 'uploader_id': 'openstreetmapus',
86 'uploader': 'OpenStreetMap US',
87 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
91 'url': 'http://player.vimeo.com/video/54469442',
92 'file': '54469442.mp4',
93 'md5': '619b811a4417aa4abe78dc653becf511',
94 'note': 'Videos that embed the url in the player page',
96 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software',
97 'uploader': 'The BLN & Business of Software',
98 'uploader_id': 'theblnbusinessofsoftware',
102 'url': 'http://vimeo.com/68375962',
103 'file': '68375962.mp4',
104 'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
105 'note': 'Video protected with password',
107 'title': 'youtube-dl password protected test video',
108 'upload_date': '20130614',
109 'uploader_id': 'user18948128',
110 'uploader': 'Jaime Marquínez Ferrándiz',
113 'videopassword': 'youtube-dl',
117 'url': 'http://vimeo.com/76979871',
118 'md5': '3363dd6ffebe3784d56f4132317fd446',
119 'note': 'Video with subtitles',
123 'title': 'The New Vimeo Player (You Know, For Videos)',
124 'description': 'md5:2ec900bf97c3f389378a96aee11260ea',
125 'upload_date': '20131015',
126 'uploader_id': 'staff',
127 'uploader': 'Vimeo Staff',
133 def suitable(cls
, url
):
134 if VimeoChannelIE
.suitable(url
):
135 # Otherwise channel urls like http://vimeo.com/channels/31259 would
139 return super(VimeoIE
, cls
).suitable(url
)
141 def _verify_video_password(self
, url
, video_id
, webpage
):
142 password
= self
._downloader
.params
.get('videopassword', None)
144 raise ExtractorError('This video is protected by a password, use the --video-password option')
145 token
= self
._search
_regex
(r
'xsrft: \'(.*?
)\'', webpage, 'login token
')
146 data = compat_urllib_parse.urlencode({'password
': password,
148 # I didn't manage to use the password
with https
149 if url
.startswith('https'):
150 pass_url
= url
.replace('https','http')
153 password_request
= compat_urllib_request
.Request(pass_url
+'/password', data
)
154 password_request
.add_header('Content-Type', 'application/x-www-form-urlencoded')
155 password_request
.add_header('Cookie', 'xsrft=%s' % token
)
156 self
._download
_webpage
(password_request
, video_id
,
157 'Verifying the password',
160 def _verify_player_video_password(self
, url
, video_id
):
161 password
= self
._downloader
.params
.get('videopassword', None)
163 raise ExtractorError('This video is protected by a password, use the --video-password option')
164 data
= compat_urllib_parse
.urlencode({'password': password}
)
165 pass_url
= url
+ '/check-password'
166 password_request
= compat_urllib_request
.Request(pass_url
, data
)
167 password_request
.add_header('Content-Type', 'application/x-www-form-urlencoded')
168 return self
._download
_json
(
169 password_request
, video_id
,
170 'Verifying the password',
173 def _real_initialize(self
):
176 def _real_extract(self
, url
):
177 url
, data
= unsmuggle_url(url
)
178 headers
= std_headers
180 headers
= headers
.copy()
183 # Extract ID from URL
184 mobj
= re
.match(self
._VALID
_URL
, url
)
185 video_id
= mobj
.group('id')
186 if mobj
.group('pro') or mobj
.group('player'):
187 url
= 'http://player.vimeo.com/video/' + video_id
189 url
= 'https://vimeo.com/' + video_id
191 # Retrieve video webpage to extract further information
192 request
= compat_urllib_request
.Request(url
, None, headers
)
194 webpage
= self
._download
_webpage
(request
, video_id
)
195 except ExtractorError
as ee
:
196 if isinstance(ee
.cause
, compat_HTTPError
) and ee
.cause
.code
== 403:
197 errmsg
= ee
.cause
.read()
198 if b
'Because of its privacy settings, this video cannot be played here' in errmsg
:
199 raise ExtractorError(
200 'Cannot download embed-only video without embedding '
201 'URL. Please call youtube-dl with the URL of the page '
202 'that embeds this video.',
206 # Now we begin extracting as much information as we can from what we
207 # retrieved. First we extract the information common to all extractors,
208 # and latter we extract those that are Vimeo specific.
209 self
.report_extraction(video_id
)
211 # Extract the config JSON
214 config_url
= self
._html
_search
_regex
(
215 r
' data-config-url="(.+?)"', webpage
, 'config URL')
216 config_json
= self
._download
_webpage
(config_url
, video_id
)
217 config
= json
.loads(config_json
)
218 except RegexNotFoundError
:
219 # For pro videos or player.vimeo.com urls
220 # We try to find out to which variable is assigned the config dic
221 m_variable_name
= re
.search('(\w)\.video\.id', webpage
)
222 if m_variable_name
is not None:
223 config_re
= r
'%s=({.+?});' % re
.escape(m_variable_name
.group(1))
225 config_re
= [r
' = {config:({.+?}),assets:', r
'(?:[abc])=({.+?});']
226 config
= self
._search
_regex
(config_re
, webpage
, 'info section',
228 config
= json
.loads(config
)
229 except Exception as e
:
230 if re
.search('The creator of this video has not given you permission to embed it on this domain.', webpage
):
231 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
233 if re
.search('<form[^>]+?id="pw_form"', webpage
) is not None:
234 self
._verify
_video
_password
(url
, video_id
, webpage
)
235 return self
._real
_extract
(url
)
237 raise ExtractorError('Unable to extract info section',
240 if config
.get('view') == 4:
241 config
= self
._verify
_player
_video
_password
(url
, video_id
)
244 video_title
= config
["video"]["title"]
246 # Extract uploader and uploader_id
247 video_uploader
= config
["video"]["owner"]["name"]
248 video_uploader_id
= config
["video"]["owner"]["url"].split('/')[-1] if config
["video"]["owner"]["url"] else None
250 # Extract video thumbnail
251 video_thumbnail
= config
["video"].get("thumbnail")
252 if video_thumbnail
is None:
253 video_thumbs
= config
["video"].get("thumbs")
254 if video_thumbs
and isinstance(video_thumbs
, dict):
255 _
, video_thumbnail
= sorted((int(width
), t_url
) for (width
, t_url
) in video_thumbs
.items())[-1]
257 # Extract video description
258 video_description
= None
260 video_description
= get_element_by_attribute("itemprop", "description", webpage
)
261 if video_description
: video_description
= clean_html(video_description
)
262 except AssertionError as err
:
263 # On some pages like (http://player.vimeo.com/video/54469442) the
264 # html tags are not closed, python 2.6 cannot handle it
265 if err
.args
[0] == 'we should not get here!':
270 # Extract upload date
271 video_upload_date
= None
272 mobj
= re
.search(r
'<meta itemprop="dateCreated" content="(\d{4})-(\d{2})-(\d{2})T', webpage
)
274 video_upload_date
= mobj
.group(1) + mobj
.group(2) + mobj
.group(3)
277 view_count
= int(self
._search
_regex
(r
'UserPlays:(\d+)', webpage
, 'view count'))
278 like_count
= int(self
._search
_regex
(r
'UserLikes:(\d+)', webpage
, 'like count'))
279 comment_count
= int(self
._search
_regex
(r
'UserComments:(\d+)', webpage
, 'comment count'))
280 except RegexNotFoundError
:
281 # This info is only available in vimeo.com/{id} urls
286 # Vimeo specific: extract request signature and timestamp
287 sig
= config
['request']['signature']
288 timestamp
= config
['request']['timestamp']
290 # Vimeo specific: extract video codec and quality information
291 # First consider quality, then codecs, then take everything
292 codecs
= [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
293 files
= {'hd': [], 'sd': [], 'other': []}
294 config_files
= config
["video"].get("files") or config
["request"].get("files")
295 for codec_name
, codec_extension
in codecs
:
296 for quality
in config_files
.get(codec_name
, []):
297 format_id
= '-'.join((codec_name
, quality
)).lower()
298 key
= quality
if quality
in files
else 'other'
300 if isinstance(config_files
[codec_name
], dict):
301 file_info
= config_files
[codec_name
][quality
]
302 video_url
= file_info
.get('url')
305 if video_url
is None:
306 video_url
= "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
307 %(video_id
, sig
, timestamp
, quality
, codec_name
.upper())
310 'ext': codec_extension
,
312 'format_id': format_id
,
313 'width': file_info
.get('width'),
314 'height': file_info
.get('height'),
317 for key
in ('other', 'sd', 'hd'):
318 formats
+= files
[key
]
319 if len(formats
) == 0:
320 raise ExtractorError('No known codec found')
323 text_tracks
= config
['request'].get('text_tracks')
325 for tt
in text_tracks
:
326 subtitles
[tt
['lang']] = 'http://vimeo.com' + tt
['url']
328 video_subtitles
= self
.extract_subtitles(video_id
, subtitles
)
329 if self
._downloader
.params
.get('listsubtitles', False):
330 self
._list
_available
_subtitles
(video_id
, subtitles
)
335 'uploader': video_uploader
,
336 'uploader_id': video_uploader_id
,
337 'upload_date': video_upload_date
,
338 'title': video_title
,
339 'thumbnail': video_thumbnail
,
340 'description': video_description
,
343 'view_count': view_count
,
344 'like_count': like_count
,
345 'comment_count': comment_count
,
346 'subtitles': video_subtitles
,
350 class VimeoChannelIE(InfoExtractor
):
351 IE_NAME
= 'vimeo:channel'
352 _VALID_URL
= r
'(?:https?://)?vimeo\.com/channels/(?P<id>[^/]+)/?(\?.*)?$'
353 _MORE_PAGES_INDICATOR
= r
'<a.+?rel="next"'
354 _TITLE_RE
= r
'<link rel="alternate"[^>]+?title="(.*?)"'
356 def _page_url(self
, base_url
, pagenum
):
357 return '%s/videos/page:%d/' % (base_url
, pagenum
)
359 def _extract_list_title(self
, webpage
):
360 return self
._html
_search
_regex
(self
._TITLE
_RE
, webpage
, 'list title')
362 def _extract_videos(self
, list_id
, base_url
):
364 for pagenum
in itertools
.count(1):
365 webpage
= self
._download
_webpage
(
366 self
._page
_url
(base_url
, pagenum
) ,list_id
,
367 'Downloading page %s' % pagenum
)
368 video_ids
.extend(re
.findall(r
'id="clip_(\d+?)"', webpage
))
369 if re
.search(self
._MORE
_PAGES
_INDICATOR
, webpage
, re
.DOTALL
) is None:
372 entries
= [self
.url_result('http://vimeo.com/%s' % video_id
, 'Vimeo')
373 for video_id
in video_ids
]
374 return {'_type': 'playlist',
376 'title': self
._extract
_list
_title
(webpage
),
380 def _real_extract(self
, url
):
381 mobj
= re
.match(self
._VALID
_URL
, url
)
382 channel_id
= mobj
.group('id')
383 return self
._extract
_videos
(channel_id
, 'http://vimeo.com/channels/%s' % channel_id
)
386 class VimeoUserIE(VimeoChannelIE
):
387 IE_NAME
= 'vimeo:user'
388 _VALID_URL
= r
'(?:https?://)?vimeo\.com/(?P<name>[^/]+)(?:/videos|[#?]|$)'
389 _TITLE_RE
= r
'<a[^>]+?class="user">([^<>]+?)</a>'
392 def suitable(cls
, url
):
393 if VimeoChannelIE
.suitable(url
) or VimeoIE
.suitable(url
) or VimeoAlbumIE
.suitable(url
) or VimeoGroupsIE
.suitable(url
):
395 return super(VimeoUserIE
, cls
).suitable(url
)
397 def _real_extract(self
, url
):
398 mobj
= re
.match(self
._VALID
_URL
, url
)
399 name
= mobj
.group('name')
400 return self
._extract
_videos
(name
, 'http://vimeo.com/%s' % name
)
403 class VimeoAlbumIE(VimeoChannelIE
):
404 IE_NAME
= 'vimeo:album'
405 _VALID_URL
= r
'(?:https?://)?vimeo\.com/album/(?P<id>\d+)'
406 _TITLE_RE
= r
'<header id="page_header">\n\s*<h1>(.*?)</h1>'
408 def _page_url(self
, base_url
, pagenum
):
409 return '%s/page:%d/' % (base_url
, pagenum
)
411 def _real_extract(self
, url
):
412 mobj
= re
.match(self
._VALID
_URL
, url
)
413 album_id
= mobj
.group('id')
414 return self
._extract
_videos
(album_id
, 'http://vimeo.com/album/%s' % album_id
)
417 class VimeoGroupsIE(VimeoAlbumIE
):
418 IE_NAME
= 'vimeo:group'
419 _VALID_URL
= r
'(?:https?://)?vimeo\.com/groups/(?P<name>[^/]+)'
421 def _extract_list_title(self
, webpage
):
422 return self
._og
_search
_title
(webpage
)
424 def _real_extract(self
, url
):
425 mobj
= re
.match(self
._VALID
_URL
, url
)
426 name
= mobj
.group('name')
427 return self
._extract
_videos
(name
, 'http://vimeo.com/groups/%s' % name
)
430 class VimeoReviewIE(InfoExtractor
):
431 IE_NAME
= 'vimeo:review'
432 IE_DESC
= 'Review pages on vimeo'
433 _VALID_URL
= r
'(?:https?://)?vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
435 'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
436 'file': '75524534.mp4',
437 'md5': 'c507a72f780cacc12b2248bb4006d253',
439 'title': "DICK HARDWICK 'Comedian'",
440 'uploader': 'Richard Hardwick',
444 def _real_extract(self
, url
):
445 mobj
= re
.match(self
._VALID
_URL
, url
)
446 video_id
= mobj
.group('id')
447 player_url
= 'https://player.vimeo.com/player/' + video_id
448 return self
.url_result(player_url
, 'Vimeo', video_id
)
451 class VimeoWatchLaterIE(VimeoBaseInfoExtractor
, VimeoChannelIE
):
452 IE_NAME
= 'vimeo:watchlater'
453 IE_DESC
= 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
454 _VALID_URL
= r
'https?://vimeo\.com/home/watchlater|:vimeowatchlater'
455 _LOGIN_REQUIRED
= True
456 _TITLE_RE
= r
'href="/home/watchlater".*?>(.*?)<'
458 def _real_initialize(self
):
461 def _page_url(self
, base_url
, pagenum
):
462 url
= '%s/page:%d/' % (base_url
, pagenum
)
463 request
= compat_urllib_request
.Request(url
)
464 # Set the header to get a partial html page with the ids,
465 # the normal page doesn't contain them.
466 request
.add_header('X-Requested-With', 'XMLHttpRequest')
469 def _real_extract(self
, url
):
470 return self
._extract
_videos
('watchlater', 'https://vimeo.com/home/watchlater')