2 from __future__
import unicode_literals
8 from .common
import InfoExtractor
12 compat_urllib_request
,
28 class VimeoBaseInfoExtractor(InfoExtractor
):
29 _NETRC_MACHINE
= 'vimeo'
30 _LOGIN_REQUIRED
= False
33 (username
, password
) = self
._get
_login
_info
()
35 if self
._LOGIN
_REQUIRED
:
36 raise ExtractorError('No login info available, needed for using %s.' % self
.IE_NAME
, expected
=True)
39 login_url
= 'https://vimeo.com/log_in'
40 webpage
= self
._download
_webpage
(login_url
, None, False)
41 token
= self
._search
_regex
(r
'xsrft = \'(.*?
)\'', webpage, 'login token
')
42 data = urlencode_postdata({
49 login_request = compat_urllib_request.Request(login_url, data)
50 login_request.add_header('Content
-Type
', 'application
/x
-www
-form
-urlencoded
')
51 login_request.add_header('Cookie
', 'xsrft
=%s' % token)
52 self._download_webpage(login_request, None, False, 'Wrong login info
')
55 class VimeoIE(VimeoBaseInfoExtractor):
56 """Information extractor for vimeo.com."""
58 # _VALID_URL matches Vimeo URLs
61 (?:(?:www|(?P<player>player))\.)?
62 vimeo(?P<pro>pro)?\.com/
63 (?!channels/[^/?#]+/?(?:$|[?#])|album/)
65 (?:(?:play_redirect_hls|moogaloop\.swf)\?clip_id=)?
68 /?(?:[?&].*)?(?:[#].*)?$'''
72 'url
': 'http
://vimeo
.com
/56015672#at=0',
73 'md5': '8879b6cc097e987f02484baf890129e5',
77 "upload_date": "20121220",
78 "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
79 "uploader_id": "user7108434",
80 "uploader": "Filippo Valsorda",
81 "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
86 'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
87 'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
88 'note': 'Vimeo Pro video (#1197)',
92 'uploader_id': 'openstreetmapus',
93 'uploader': 'OpenStreetMap US',
94 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
95 'description': 'md5:380943ec71b89736ff4bf27183233d09',
100 'url': 'http://player.vimeo.com/video/54469442',
101 'md5': '619b811a4417aa4abe78dc653becf511',
102 'note': 'Videos that embed the url in the player page',
106 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software 2012',
107 'uploader': 'The BLN & Business of Software',
108 'uploader_id': 'theblnbusinessofsoftware',
114 'url': 'http://vimeo.com/68375962',
115 'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
116 'note': 'Video protected with password',
120 'title': 'youtube-dl password protected test video',
121 'upload_date': '20130614',
122 'uploader_id': 'user18948128',
123 'uploader': 'Jaime Marquínez Ferrándiz',
125 'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people who love them.',
128 'videopassword': 'youtube-dl',
132 'url': 'http://vimeo.com/channels/keypeele/75629013',
133 'md5': '2f86a05afe9d7abc0b9126d229bbe15d',
134 'note': 'Video is freely available via original URL '
135 'and protected with password when accessed via http://vimeo.com/75629013',
139 'title': 'Key & Peele: Terrorist Interrogation',
140 'description': 'md5:8678b246399b070816b12313e8b4eb5c',
141 'uploader_id': 'atencio',
142 'uploader': 'Peter Atencio',
143 'upload_date': '20130927',
148 'url': 'http://vimeo.com/76979871',
149 'md5': '3363dd6ffebe3784d56f4132317fd446',
150 'note': 'Video with subtitles',
154 'title': 'The New Vimeo Player (You Know, For Videos)',
155 'description': 'md5:2ec900bf97c3f389378a96aee11260ea',
156 'upload_date': '20131015',
157 'uploader_id': 'staff',
158 'uploader': 'Vimeo Staff',
163 # from https://www.ouya.tv/game/Pier-Solar-and-the-Great-Architects/
164 'url': 'https://player.vimeo.com/video/98044508',
165 'note': 'The js code contains assignments to the same variable as the config',
169 'title': 'Pier Solar OUYA Official Trailer',
170 'uploader': 'Tulio Gonçalves',
171 'uploader_id': 'user28849593',
176 def _verify_video_password(self
, url
, video_id
, webpage
):
177 password
= self
._downloader
.params
.get('videopassword', None)
179 raise ExtractorError('This video is protected by a password, use the --video-password option', expected
=True)
180 token
= self
._search
_regex
(r
'xsrft = \'(.*?
)\'', webpage, 'login token
')
181 data = urlencode_postdata({
182 'password
': password,
185 if url.startswith('http
://'):
186 # vimeo only supports https now, but the user can give an http url
187 url = url.replace('http
://', 'https
://')
188 password_request = compat_urllib_request.Request(url + '/password
', data)
189 password_request.add_header('Content
-Type
', 'application
/x
-www
-form
-urlencoded
')
190 password_request.add_header('Cookie
', 'xsrft
=%s' % token)
191 return self._download_webpage(
192 password_request, video_id,
193 'Verifying the password
', 'Wrong password
')
195 def _verify_player_video_password(self, url, video_id):
196 password = self._downloader.params.get('videopassword
', None)
198 raise ExtractorError('This video
is protected by a password
, use the
--video
-password option
')
199 data = compat_urllib_parse.urlencode({'password': password})
200 pass_url = url + '/check
-password
'
201 password_request = compat_urllib_request.Request(pass_url, data)
202 password_request.add_header('Content
-Type
', 'application
/x
-www
-form
-urlencoded
')
203 return self._download_json(
204 password_request, video_id,
205 'Verifying the password
',
208 def _real_initialize(self):
211 def _real_extract(self, url):
212 url, data = unsmuggle_url(url)
213 headers = std_headers
215 headers = headers.copy()
217 if 'Referer
' not in headers:
218 headers['Referer
'] = url
220 # Extract ID from URL
221 mobj = re.match(self._VALID_URL, url)
222 video_id = mobj.group('id')
224 if mobj.group('pro
') or mobj.group('player
'):
225 url = 'https
://player
.vimeo
.com
/video
/' + video_id
227 # Retrieve video webpage to extract further information
228 request = compat_urllib_request.Request(url, None, headers)
230 webpage = self._download_webpage(request, video_id)
231 except ExtractorError as ee:
232 if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
233 errmsg = ee.cause.read()
234 if b'Because of its privacy settings
, this video cannot be played here
' in errmsg:
235 raise ExtractorError(
236 'Cannot download embed
-only video without embedding
'
237 'URL
. Please call youtube
-dl
with the URL of the page
'
238 'that embeds this video
.',
242 # Now we begin extracting as much information as we can from what we
243 # retrieved. First we extract the information common to all extractors,
244 # and latter we extract those that are Vimeo specific.
245 self.report_extraction(video_id)
247 # Extract the config JSON
250 config_url = self._html_search_regex(
251 r' data
-config
-url
="(.+?)"', webpage, 'config URL
')
252 config_json = self._download_webpage(config_url, video_id)
253 config = json.loads(config_json)
254 except RegexNotFoundError:
255 # For pro videos or player.vimeo.com urls
256 # We try to find out to which variable is assigned the config dic
257 m_variable_name = re.search('(\w
)\
.video\
.id', webpage)
258 if m_variable_name is not None:
259 config_re = r'%s=({[^}
].+?
});' % re.escape(m_variable_name.group(1))
261 config_re = [r' = {config:({.+?}
),assets
:', r'(?
:[abc
])=({.+?}
);']
262 config = self._search_regex(config_re, webpage, 'info section
',
264 config = json.loads(config)
265 except Exception as e:
266 if re.search('The creator of this video has
not given you permission to embed it on this domain
.', webpage):
267 raise ExtractorError('The author has restricted the access to this video
, try with the
"--referer" option
')
269 if re.search(r'<form
[^
>]+?
id="pw_form"', webpage) is not None:
270 if data and '_video_password_verified
' in data:
271 raise ExtractorError('video password verification failed
!')
272 self._verify_video_password(url, video_id, webpage)
273 return self._real_extract(
274 smuggle_url(url, {'_video_password_verified': 'verified'}))
276 raise ExtractorError('Unable to extract info section
',
279 if config.get('view
') == 4:
280 config = self._verify_player_video_password(url, video_id)
283 video_title = config["video"]["title"]
285 # Extract uploader and uploader_id
286 video_uploader = config["video"]["owner"]["name"]
287 video_uploader_id = config["video"]["owner"]["url"].split('/')[-1] if config["video"]["owner"]["url"] else None
289 # Extract video thumbnail
290 video_thumbnail = config["video"].get("thumbnail")
291 if video_thumbnail is None:
292 video_thumbs = config["video"].get("thumbs")
293 if video_thumbs and isinstance(video_thumbs, dict):
294 _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
296 # Extract video description
298 video_description = self._html_search_regex(
299 r'(?s
)<div\s
+class="[^"]*description
[^
"]*"[^
>]*>(.*?
)</div
>',
300 webpage, 'description
', default=None)
301 if not video_description:
302 video_description = self._html_search_meta(
303 'description
', webpage, default=None)
304 if not video_description and mobj.group('pro
'):
305 orig_webpage = self._download_webpage(
307 note='Downloading webpage
for description
',
310 video_description = self._html_search_meta(
311 'description
', orig_webpage, default=None)
312 if not video_description and not mobj.group('player
'):
313 self._downloader.report_warning('Cannot find video description
')
315 # Extract video duration
316 video_duration = int_or_none(config["video"].get("duration"))
318 # Extract upload date
319 video_upload_date = None
320 mobj = re.search(r'<time
[^
>]+datetime
="([^"]+)"', webpage)
322 video_upload_date = unified_strdate(mobj.group(1))
325 view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
326 like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, 'like count'))
327 comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, 'comment count'))
328 except RegexNotFoundError:
329 # This info is only available in vimeo.com/{id} urls
334 # Vimeo specific: extract request signature and timestamp
335 sig = config['request']['signature']
336 timestamp = config['request']['timestamp']
338 # Vimeo specific: extract video codec and quality information
339 # First consider quality, then codecs, then take everything
340 codecs = [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
341 files = {'hd': [], 'sd': [], 'other': []}
342 config_files = config["video
"].get("files
") or config["request
"].get("files
")
343 for codec_name, codec_extension in codecs:
344 for quality in config_files.get(codec_name, []):
345 format_id = '-'.join((codec_name, quality)).lower()
346 key = quality if quality in files else 'other'
348 if isinstance(config_files[codec_name], dict):
349 file_info = config_files[codec_name][quality]
350 video_url = file_info.get('url')
353 if video_url is None:
354 video_url = "http
://player
.vimeo
.com
/play_redirect?clip_id
=%s&sig
=%s&time
=%s&quality
=%s&codecs
=%s&type=moogaloop_local
&embed_location
=" \
355 % (video_id, sig, timestamp, quality, codec_name.upper())
358 'ext': codec_extension,
360 'format_id': format_id,
361 'width': file_info.get('width'),
362 'height': file_info.get('height'),
365 for key in ('other', 'sd', 'hd'):
366 formats += files[key]
367 if len(formats) == 0:
368 raise ExtractorError('No known codec found')
371 text_tracks = config['request'].get('text_tracks')
373 for tt in text_tracks:
374 subtitles[tt['lang']] = [{
376 'url': 'https://vimeo.com' + tt['url'],
381 'uploader': video_uploader,
382 'uploader_id': video_uploader_id,
383 'upload_date': video_upload_date,
384 'title': video_title,
385 'thumbnail': video_thumbnail,
386 'description': video_description,
387 'duration': video_duration,
390 'view_count': view_count,
391 'like_count': like_count,
392 'comment_count': comment_count,
393 'subtitles': subtitles,
397 class VimeoChannelIE(InfoExtractor):
398 IE_NAME = 'vimeo:channel'
399 _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
400 _MORE_PAGES_INDICATOR = r'<a.+?rel="next
"'
401 _TITLE_RE = r'<link rel="alternate
"[^>]+?title="(.*?
)"'
403 'url': 'https://vimeo.com/channels/tributes',
406 'title': 'Vimeo Tributes',
408 'playlist_mincount': 25,
411 def _page_url(self, base_url, pagenum):
412 return '%s/videos/page:%d/' % (base_url, pagenum)
414 def _extract_list_title(self, webpage):
415 return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
417 def _login_list_password(self, page_url, list_id, webpage):
418 login_form = self._search_regex(
419 r'(?s)<form[^>]+?id="pw_form
"(.*?)</form>',
420 webpage, 'login form', default=None)
424 password = self._downloader.params.get('videopassword', None)
426 raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
427 fields = dict(re.findall(r'''(?x)<input\s+
432 token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
433 fields['token'] = token
434 fields['password'] = password
435 post = urlencode_postdata(fields)
436 password_path = self._search_regex(
437 r'action="([^
"]+)"', login_form, 'password URL
')
438 password_url = compat_urlparse.urljoin(page_url, password_path)
439 password_request = compat_urllib_request.Request(password_url, post)
440 password_request.add_header('Content
-type', 'application
/x
-www
-form
-urlencoded
')
441 self._set_cookie('vimeo
.com
', 'xsrft
', token)
443 return self._download_webpage(
444 password_request, list_id,
445 'Verifying the password
', 'Wrong password
')
447 def _extract_videos(self, list_id, base_url):
449 for pagenum in itertools.count(1):
450 page_url = self._page_url(base_url, pagenum)
451 webpage = self._download_webpage(
453 'Downloading page
%s' % pagenum)
456 webpage = self._login_list_password(page_url, list_id, webpage)
458 video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
459 if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
462 entries = [self.url_result('https
://vimeo
.com
/%s' % video_id, 'Vimeo
')
463 for video_id in video_ids]
464 return {'_type
': 'playlist
',
466 'title
': self._extract_list_title(webpage),
470 def _real_extract(self, url):
471 mobj = re.match(self._VALID_URL, url)
472 channel_id = mobj.group('id')
473 return self._extract_videos(channel_id, 'https
://vimeo
.com
/channels
/%s' % channel_id)
476 class VimeoUserIE(VimeoChannelIE):
477 IE_NAME = 'vimeo
:user
'
478 _VALID_URL = r'https
://vimeo\
.com
/(?
![0-9]+(?
:$|
[?
#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
479 _TITLE_RE
= r
'<a[^>]+?class="user">([^<>]+?)</a>'
481 'url': 'https://vimeo.com/nkistudio/videos',
486 'playlist_mincount': 66,
489 def _real_extract(self
, url
):
490 mobj
= re
.match(self
._VALID
_URL
, url
)
491 name
= mobj
.group('name')
492 return self
._extract
_videos
(name
, 'https://vimeo.com/%s' % name
)
495 class VimeoAlbumIE(VimeoChannelIE
):
496 IE_NAME
= 'vimeo:album'
497 _VALID_URL
= r
'https://vimeo\.com/album/(?P<id>\d+)'
498 _TITLE_RE
= r
'<header id="page_header">\n\s*<h1>(.*?)</h1>'
500 'url': 'https://vimeo.com/album/2632481',
503 'title': 'Staff Favorites: November 2013',
505 'playlist_mincount': 13,
507 'note': 'Password-protected album',
508 'url': 'https://vimeo.com/album/3253534',
515 'videopassword': 'youtube-dl',
519 def _page_url(self
, base_url
, pagenum
):
520 return '%s/page:%d/' % (base_url
, pagenum
)
522 def _real_extract(self
, url
):
523 album_id
= self
._match
_id
(url
)
524 return self
._extract
_videos
(album_id
, 'https://vimeo.com/album/%s' % album_id
)
527 class VimeoGroupsIE(VimeoAlbumIE
):
528 IE_NAME
= 'vimeo:group'
529 _VALID_URL
= r
'https://vimeo\.com/groups/(?P<name>[^/]+)'
531 'url': 'https://vimeo.com/groups/rolexawards',
534 'title': 'Rolex Awards for Enterprise',
536 'playlist_mincount': 73,
539 def _extract_list_title(self
, webpage
):
540 return self
._og
_search
_title
(webpage
)
542 def _real_extract(self
, url
):
543 mobj
= re
.match(self
._VALID
_URL
, url
)
544 name
= mobj
.group('name')
545 return self
._extract
_videos
(name
, 'https://vimeo.com/groups/%s' % name
)
548 class VimeoReviewIE(InfoExtractor
):
549 IE_NAME
= 'vimeo:review'
550 IE_DESC
= 'Review pages on vimeo'
551 _VALID_URL
= r
'https://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
553 'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
554 'md5': 'c507a72f780cacc12b2248bb4006d253',
558 'title': "DICK HARDWICK 'Comedian'",
559 'uploader': 'Richard Hardwick',
562 'note': 'video player needs Referer',
563 'url': 'https://vimeo.com/user22258446/review/91613211/13f927e053',
564 'md5': '6295fdab8f4bf6a002d058b2c6dce276',
568 'title': 're:(?i)^Death by dogma versus assembling agile . Sander Hoogendoorn',
569 'uploader': 'DevWeek Events',
571 'thumbnail': 're:^https?://.*\.jpg$',
575 def _real_extract(self
, url
):
576 mobj
= re
.match(self
._VALID
_URL
, url
)
577 video_id
= mobj
.group('id')
578 player_url
= 'https://player.vimeo.com/player/' + video_id
579 return self
.url_result(player_url
, 'Vimeo', video_id
)
582 class VimeoWatchLaterIE(VimeoBaseInfoExtractor
, VimeoChannelIE
):
583 IE_NAME
= 'vimeo:watchlater'
584 IE_DESC
= 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
585 _VALID_URL
= r
'https://vimeo\.com/home/watchlater|:vimeowatchlater'
586 _LOGIN_REQUIRED
= True
587 _TITLE_RE
= r
'href="/home/watchlater".*?>(.*?)<'
589 'url': 'https://vimeo.com/home/watchlater',
590 'only_matching': True,
593 def _real_initialize(self
):
596 def _page_url(self
, base_url
, pagenum
):
597 url
= '%s/page:%d/' % (base_url
, pagenum
)
598 request
= compat_urllib_request
.Request(url
)
599 # Set the header to get a partial html page with the ids,
600 # the normal page doesn't contain them.
601 request
.add_header('X-Requested-With', 'XMLHttpRequest')
604 def _real_extract(self
, url
):
605 return self
._extract
_videos
('watchlater', 'https://vimeo.com/home/watchlater')
608 class VimeoLikesIE(InfoExtractor
):
609 _VALID_URL
= r
'https://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes/?(?:$|[?#]|sort:)'
610 IE_NAME
= 'vimeo:likes'
611 IE_DESC
= 'Vimeo user likes'
613 'url': 'https://vimeo.com/user755559/likes/',
614 'playlist_mincount': 293,
616 'id': 'user755559_likes',
617 "description": "See all the videos urza likes",
618 "title": 'Videos urza likes',
622 def _real_extract(self
, url
):
623 user_id
= self
._match
_id
(url
)
624 webpage
= self
._download
_webpage
(url
, user_id
)
625 page_count
= self
._int
(
627 r
'''(?x)<li><a\s+href="[^"]+"\s+data-page="([0-9]+)">
628 .*?</a></li>\s*<li\s+class="pagination_next">
629 ''', webpage
, 'page count'),
630 'page count', fatal
=True)
632 title
= self
._html
_search
_regex
(
633 r
'(?s)<h1>(.+?)</h1>', webpage
, 'title', fatal
=False)
634 description
= self
._html
_search
_meta
('description', webpage
)
637 page_url
= 'https://vimeo.com/user%s/likes/page:%d/sort:date' % (
639 webpage
= self
._download
_webpage
(
641 note
='Downloading page %d/%d' % (idx
+ 1, page_count
))
642 video_list
= self
._search
_regex
(
643 r
'(?s)<ol class="js-browse_list[^"]+"[^>]*>(.*?)</ol>',
644 webpage
, 'video content')
646 r
'<li[^>]*>\s*<a\s+href="([^"]+)"', video_list
)
650 'url': compat_urlparse
.urljoin(page_url
, path
),
653 pl
= InAdvancePagedList(_get_page
, page_count
, PAGE_SIZE
)
657 'id': 'user%s_likes' % user_id
,
659 'description': description
,