5 from .common
import InfoExtractor
6 from .dailymotion
import DailymotionIE
7 from .odnoklassniki
import OdnoklassnikiIE
8 from .pladform
import PladformIE
9 from .vimeo
import VimeoIE
10 from .youtube
import YoutubeIE
11 from ..compat
import compat_urlparse
28 class VKBaseIE(InfoExtractor
):
31 def _download_webpage_handle(self
, url_or_request
, video_id
, *args
, fatal
=True, **kwargs
):
32 response
= super()._download
_webpage
_handle
(url_or_request
, video_id
, *args
, fatal
=fatal
, **kwargs
)
33 challenge_url
, cookie
= response
[1].geturl() if response
else '', None
34 if challenge_url
.startswith('https://vk.com/429.html?'):
35 cookie
= self
._get
_cookies
(challenge_url
).get('hash429')
39 hash429
= hashlib
.md5(cookie
.value
.encode('ascii')).hexdigest()
40 self
._request
_webpage
(
41 update_url_query(challenge_url
, {'key': hash429}
), video_id
, fatal
=fatal
,
42 note
='Resolving WAF challenge', errnote
='Failed to bypass WAF challenge')
43 return super()._download
_webpage
_handle
(url_or_request
, video_id
, *args
, fatal
=True, **kwargs
)
45 def _perform_login(self
, username
, password
):
46 login_page
, url_handle
= self
._download
_webpage
_handle
(
47 'https://vk.com', None, 'Downloading login page')
49 login_form
= self
._hidden
_inputs
(login_page
)
52 'email': username
.encode('cp1251'),
53 'pass': password
.encode('cp1251'),
56 # vk serves two same remixlhk cookies in Set-Cookie header and expects
57 # first one to be actually set
58 self
._apply
_first
_set
_cookie
_header
(url_handle
, 'remixlhk')
60 login_page
= self
._download
_webpage
(
61 'https://vk.com/login', None,
63 data
=urlencode_postdata(login_form
))
65 if re
.search(r
'onLoginFailed', login_page
):
67 'Unable to login, incorrect username and/or password', expected
=True)
69 def _download_payload(self
, path
, video_id
, data
, fatal
=True):
70 endpoint
= f
'https://vk.com/{path}.php'
72 code
, payload
= self
._download
_json
(
73 endpoint
, video_id
, data
=urlencode_postdata(data
), fatal
=fatal
,
76 'X-Requested-With': 'XMLHttpRequest',
79 self
.raise_login_required()
81 raise ExtractorError(clean_html(payload
[0][1:-1]), expected
=True)
92 (?:(?:m|new)\.)?vk\.com/video_|
95 ext\.php\?(?P<embed_query>.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+).*)|
97 (?:(?:m|new)\.)?vk\.com/(?:.+?\?.*?z=)?(?:video|clip)|
98 (?:www\.)?daxab.com/embed/
100 (?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>([\da-f]+)|(ln-[\da-zA-Z]+)))?
105 'url': 'http://vk.com/videos-77521?z=video-77521_162222515%2Fclub77521',
107 'id': '-77521_162222515',
109 'title': 'ProtivoGunz - Хуёвая песня',
110 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
111 'uploader_id': '39545378',
113 'timestamp': 1329049880,
114 'upload_date': '20120212',
115 'comment_count': int,
117 'thumbnail': r
're:https?://.+\.jpg$',
119 'params': {'skip_download': 'm3u8'}
,
122 'url': 'http://vk.com/video205387401_165548505',
124 'id': '205387401_165548505',
127 'uploader': 'Tom Cruise',
128 'uploader_id': '205387401',
130 'timestamp': 1374364108,
131 'upload_date': '20130720',
132 'comment_count': int,
134 'thumbnail': r
're:https?://.+\.jpg$',
138 'note': 'Embedded video',
139 'url': 'https://vk.com/video_ext.php?oid=-77521&id=162222515&hash=87b046504ccd8bfa',
141 'id': '-77521_162222515',
143 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
144 'title': 'ProtivoGunz - Хуёвая песня',
146 'upload_date': '20120212',
147 'timestamp': 1329049880,
148 'uploader_id': '39545378',
149 'thumbnail': r
're:https?://.+\.jpg$',
151 'params': {'skip_download': 'm3u8'}
,
155 # please update if you find a video whose URL follows the same pattern
156 'url': 'http://vk.com/video-8871596_164049491',
157 'md5': 'a590bcaf3d543576c9bd162812387666',
158 'note': 'Only available for registered users',
160 'id': '-8871596_164049491',
162 'uploader': 'Триллеры',
163 'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]',
165 'upload_date': '20121218',
171 'url': 'http://vk.com/hd_kino_mania?z=video-43215063_168067957%2F15c66b9b533119788d',
173 'id': '-43215063_168067957',
175 'uploader': 'Bro Mazter',
178 'upload_date': '20140328',
179 'uploader_id': '223413403',
180 'timestamp': 1396018030,
182 'skip': 'Requires vk account credentials',
185 'url': 'http://m.vk.com/video-43215063_169084319?list=125c627d1aa1cebb83&from=wall-43215063_2566540',
186 'md5': '0c45586baa71b7cb1d0784ee3f4e00a6',
187 'note': 'ivi.ru embed',
189 'id': '-43215063_169084319',
191 'title': 'Книга Илая',
193 'upload_date': '20140626',
199 'url': 'https://vk.com/video-93049196_456239755?list=ln-cBjJ7S4jYYx3ADnmDT',
201 'id': '-93049196_456239755',
203 'title': '8 серия (озвучка)',
205 'comment_count': int,
206 'uploader': 'Dizi2021',
208 'timestamp': 1640162189,
209 'upload_date': '20211222',
210 'uploader_id': '-93049196',
211 'thumbnail': r
're:https?://.+\.jpg$',
215 # video (removed?) only available with list id
216 'url': 'https://vk.com/video30481095_171201961?list=8764ae2d21f14088d4',
217 'md5': '091287af5402239a1051c37ec7b92913',
219 'id': '30481095_171201961',
221 'title': 'ТюменцевВВ_09.07.2015',
222 'uploader': 'Anton Ivanov',
224 'upload_date': '20150709',
231 'url': 'https://vk.com/video276849682_170681728',
235 'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate",
236 'description': 'md5:bf9c26cfa4acdfb146362682edd3827a',
238 'upload_date': '20130117',
239 'uploader': "Children's Joy Foundation Inc.",
240 'uploader_id': 'thecjf',
242 'channel_id': 'UCgzCNQ11TmR9V97ECnhi3gw',
243 'availability': 'public',
245 'live_status': 'not_live',
246 'playable_in_embed': True,
247 'channel': 'Children\'s Joy Foundation Inc.',
248 'uploader_url': 'http://www.youtube.com/user/thecjf',
249 'thumbnail': r
're:https?://.+\.jpg$',
252 'categories': ['Nonprofits & Activism'],
253 'channel_url': 'https://www.youtube.com/channel/UCgzCNQ11TmR9V97ECnhi3gw',
259 'url': 'https://vk.com/video-37468416_456239855',
261 'id': 'k3lz2cmXyRuJQSjGHUv',
263 'title': 'md5:d52606645c20b0ddbb21655adaa4f56f',
264 'description': 'md5:424b8e88cc873217f520e582ba28bb36',
265 'uploader': 'AniLibria.Tv',
266 'upload_date': '20160914',
267 'uploader_id': 'x1p5vl5',
268 'timestamp': 1473877246,
273 # video key is extra_data not url\d+
274 'url': 'http://vk.com/video-110305615_171782105',
275 'md5': 'e13fcda136f99764872e739d13fac1d1',
277 'id': '-110305615_171782105',
279 'title': 'S-Dance, репетиции к The way show',
280 'uploader': 'THE WAY SHOW | 17 апреля',
281 'uploader_id': '-110305615',
282 'timestamp': 1454859345,
283 'upload_date': '20160207',
288 # finished live stream, postlive_mp4
289 'url': 'https://vk.com/videos-387766?z=video-387766_456242764%2Fpl_-387766_-2',
291 'id': '-387766_456242764',
293 'title': 'ИгроМир 2016 День 1 — Игромания Утром',
294 'uploader': 'Игромания',
296 'upload_date': '20160929',
297 'uploader_id': '-387766',
298 'timestamp': 1475137527,
299 'thumbnail': r
're:https?://.+\.jpg$',
300 'comment_count': int,
304 'skip_download': True,
308 # live stream, hls and rtmp links, most likely already finished live
309 # stream by the time you are reading this comment
310 'url': 'https://vk.com/video-140332_456239111',
311 'only_matching': True,
314 # removed video, just testing that we match the pattern
315 'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
316 'only_matching': True,
319 # age restricted video, requires vk account credentials
320 'url': 'https://vk.com/video205387401_164765225',
321 'only_matching': True,
325 'url': 'https://vk.com/video-76116461_171554880',
326 'only_matching': True,
329 'url': 'http://new.vk.com/video205387401_165548505',
330 'only_matching': True,
333 # This video is no longer available, because its author has been blocked.
334 'url': 'https://vk.com/video-10639516_456240611',
335 'only_matching': True,
338 # The video is not available in your region.
339 'url': 'https://vk.com/video-51812607_171445436',
340 'only_matching': True,
343 'url': 'https://vk.com/clip30014565_456240946',
344 'only_matching': True,
348 def _extract_sibnet_urls(webpage
):
349 # https://help.sibnet.ru/?sibnet_video_embed
350 return [unescapeHTML(mobj
.group('url')) for mobj
in re
.finditer(
351 r
'<iframe\b[^>]+\bsrc=(["\'])(?P
<url
>(?
:https?
:)?
//video\
.sibnet\
.ru
/shell\
.php
\?.*?
\bvideoid
=\d
+.*?
)\
1',
354 def _real_extract(self, url):
355 mobj = self._match_valid_url(url)
356 video_id = mobj.group('videoid
')
364 # Some videos (removed?) can only be downloaded with list id specified
365 list_id = mobj.group('list_id
')
367 data['list'] = list_id
369 payload = self._download_payload('al_video
', video_id, data)
370 info_page = payload[1]
372 mv_data = opts.get('mvData
') or {}
373 player = opts.get('player
') or {}
375 video_id = '%s_%s' % (mobj.group('oid
'), mobj.group('id'))
377 info_page = self._download_webpage(
378 'http
://vk
.com
/video_ext
.php?
' + mobj.group('embed_query
'), video_id)
380 error_message = self._html_search_regex(
381 [r'(?s
)<!><div
[^
>]+class="video_layer_message"[^
>]*>(.+?
)</div
>',
382 r'(?s
)<div
[^
>]+id="video_ext_msg"[^
>]*>(.+?
)</div
>'],
383 info_page, 'error message
', default=None)
385 raise ExtractorError(error_message, expected=True)
387 if re.search(r'<!>/login\
.php
\?.*\bact
=security_check
', info_page):
388 raise ExtractorError(
389 'You are trying to log
in from an unusual location
. You should confirm ownership at vk
.com to log
in with this IP
.',
392 ERROR_COPYRIGHT = 'Video
%s has been removed
from public access due to rightholder complaint
.'
395 r'>Видеозапись
.*? была изъята из публичного доступа в связи с обращением правообладателя
.<':
398 r'>The video
.*? was removed
from public access by request of the copyright holder
.<':
401 r'<!>Please log
in or <':
402 'Video
%s is only available
for registered users
, '
403 'use
--username
and --password options to provide account credentials
.',
406 'Video
%s does
not exist
.',
408 r'<!>Видео временно недоступно
':
409 'Video
%s is temporarily unavailable
.',
412 'Access denied to video
%s.',
414 r'<!>Видеозапись недоступна
, так как её автор был заблокирован
.':
415 'Video
%s is no longer available
, because its author has been blocked
.',
417 r'<!>This video
is no longer available
, because its author has been blocked
.':
418 'Video
%s is no longer available
, because its author has been blocked
.',
420 r'<!>This video
is no longer available
, because it has been deleted
.':
421 'Video
%s is no longer available
, because it has been deleted
.',
423 r'<!>The video
.+?
is not available
in your region
.':
424 'Video
%s is not available
in your region
.',
427 for error_re, error_msg in ERRORS.items():
428 if re.search(error_re, info_page):
429 raise ExtractorError(error_msg % video_id, expected=True)
431 player = self._parse_json(self._search_regex(
432 r'var\s
+playerParams\s
*=\s
*({.+?}
)\s
*;\s
*\n',
433 info_page, 'player params
'), video_id)
435 youtube_url = YoutubeIE._extract_url(info_page)
437 return self.url_result(youtube_url, YoutubeIE.ie_key())
439 vimeo_url = VimeoIE._extract_url(url, info_page)
440 if vimeo_url is not None:
441 return self.url_result(vimeo_url, VimeoIE.ie_key())
443 pladform_url = PladformIE._extract_url(info_page)
445 return self.url_result(pladform_url, PladformIE.ie_key())
447 m_rutube = re.search(
448 r'\ssrc
="((?:https?:)?//rutube\.ru\\?/(?:video|play)\\?/embed(?:.*?))\\?"', info_page)
449 if m_rutube is not None:
450 rutube_url = self._proto_relative_url(
451 m_rutube.group(1).replace('\\', ''))
452 return self.url_result(rutube_url)
454 dailymotion_urls = DailymotionIE._extract_urls(info_page)
456 return self.url_result(dailymotion_urls[0], DailymotionIE.ie_key())
458 odnoklassniki_url = OdnoklassnikiIE._extract_url(info_page)
459 if odnoklassniki_url:
460 return self.url_result(odnoklassniki_url, OdnoklassnikiIE.ie_key())
462 sibnet_urls = self._extract_sibnet_urls(info_page)
464 return self.url_result(sibnet_urls[0])
466 m_opts = re.search(r'(?s
)var\s
+opts\s
*=\s
*({.+?}
);', info_page)
468 m_opts_url = re.search(r"url\s*:\s*'((?
!/\b)[^
']+)", m_opts.group(1))
470 opts_url = m_opts_url.group(1)
471 if opts_url.startswith('//'):
472 opts_url = 'http
:' + opts_url
473 return self.url_result(opts_url)
475 data = player['params
'][0]
476 title = unescapeHTML(data['md_title
'])
479 # 3 = post live (finished live)
480 is_live = data.get('live
') == 2
482 timestamp = unified_timestamp(self._html_search_regex(
483 r'class=["\']mv_info_date[^>]+>([^<]+)(?:<|from)', info_page,
484 'upload date', default=None)) or int_or_none(data.get('date'))
486 view_count = str_to_int(self._search_regex(
487 r'class=["\']mv_views_count
[^
>]+>\s
*([\d
,.]+)',
488 info_page, 'view count
', default=None))
491 for format_id, format_url in data.items():
492 format_url = url_or_none(format_url)
493 if not format_url or not format_url.startswith(('http
', '//', 'rtmp
')):
495 if (format_id.startswith(('url
', 'cache
'))
496 or format_id in ('extra_data
', 'live_mp4
', 'postlive_mp4
')):
497 height = int_or_none(self._search_regex(
498 r'^
(?
:url|cache
)(\d
+)', format_id, 'height
', default=None))
500 'format_id
': format_id,
504 elif format_id == 'hls
':
505 formats.extend(self._extract_m3u8_formats(
506 format_url, video_id, 'mp4
', 'm3u8_native
',
507 m3u8_id=format_id, fatal=False, live=is_live))
508 elif format_id == 'rtmp
':
510 'format_id
': format_id,
514 self._sort_formats(formats)
517 for sub in data.get('subs
') or {}:
518 subtitles.setdefault(sub.get('lang
', 'en
'), []).append({
519 'ext
': sub.get('title
', '.srt
').split('.')[-1],
520 'url
': url_or_none(sub.get('url
')),
527 'thumbnail
': data.get('jpg
'),
528 'uploader
': data.get('md_author
'),
529 'uploader_id
': str_or_none(data.get('author_id
') or mv_data.get('authorId
')),
530 'duration
': int_or_none(data.get('duration
') or mv_data.get('duration
')),
531 'timestamp
': timestamp,
532 'view_count
': view_count,
533 'like_count
': int_or_none(mv_data.get('likes
')),
534 'comment_count
': int_or_none(mv_data.get('commcount
')),
536 'subtitles
': subtitles,
540 class VKUserVideosIE(VKBaseIE):
541 IE_NAME = 'vk
:uservideos
'
542 IE_DESC = "VK - User's Videos
"
543 _VALID_URL = r'https?://(?:(?:m|new)\.)?vk\.com/video/@(?P<id>[^?$#/&]+)(?!\?.*\bz=video)(?:[/?#&](?:.*?\bsection=(?P<section>\w+))?|$)'
544 _TEMPLATE_URL = 'https://vk.com/videos'
546 'url': 'https://vk.com/video/@mobidevices',
548 'id': '-17892518_all',
550 'playlist_mincount': 1355,
552 'url': 'https://vk.com/video/@mobidevices?section=uploaded',
554 'id': '-17892518_uploaded',
556 'playlist_mincount': 182,
558 _VIDEO = collections.namedtuple('Video', ['owner_id', 'id'])
560 def _entries(self, page_id, section):
561 video_list_json = self._download_payload('al_video', page_id, {
562 'act': 'load_videos_silent',
567 count = video_list_json['count']
568 total = video_list_json['total']
569 video_list = video_list_json['list']
572 for video in video_list:
573 v = self._VIDEO._make(video[:2])
574 video_id = '%d_%d' % (v.owner_id, v.id)
575 yield self.url_result(
576 'http://vk.com/video' + video_id, VKIE.ie_key(), video_id)
579 video_list_json = self._download_payload('al_video', page_id, {
580 'act': 'load_videos_silent',
585 count += video_list_json['count']
586 video_list = video_list_json['list']
588 def _real_extract(self, url):
589 u_id, section = self._match_valid_url(url).groups()
590 webpage = self._download_webpage(url, u_id)
591 page_id = self._search_regex(r'data-owner-id\s?=\s?"([^
"]+)"', webpage, 'page_id
')
595 return self.playlist_result(self._entries(page_id, section), '%s_%s' % (page_id, section))
598 class VKWallPostIE(VKBaseIE):
599 IE_NAME = 'vk
:wallpost
'
600 _VALID_URL = r'https?
://(?
:(?
:(?
:(?
:m|new
)\
.)?vk\
.com
/(?
:[^?
]+\?.*\bw
=)?
wall(?P
<id>-?\d
+_\d
+)))'
602 # public page URL, audio playlist
603 'url
': 'https
://vk
.com
/bs
.official?w
=wall
-23538238_35',
605 'id': '-23538238_35',
606 'title
': 'Black Shadow
- Wall post
-23538238_35',
607 'description
': 'md5
:3f84b9c4f9ef499731cf1ced9998cc0c
',
610 'md5
': '5ba93864ec5b85f7ce19a9af4af080f6
',
612 'id': '135220665_111806521',
614 'title
': 'Black Shadow
- Слепое Верование
',
616 'uploader
': 'Black Shadow
',
617 'artist
': 'Black Shadow
',
618 'track
': 'Слепое Верование
',
621 'md5
': '4cc7e804579122b17ea95af7834c9233
',
623 'id': '135220665_111802303',
625 'title
': 'Black Shadow
- Война
- Негасимое Бездны Пламя
!',
627 'uploader
': 'Black Shadow
',
628 'artist
': 'Black Shadow
',
629 'track
': 'Война
- Негасимое Бездны Пламя
!',
633 'skip_download
': True,
635 'skip
': 'Requires vk account credentials
',
637 # single YouTube embed, no leading -
638 'url
': 'https
://vk
.com
/wall85155021_6319
',
640 'id': '85155021_6319',
641 'title
': 'Сергей Горбунов
- Wall post
85155021_6319',
644 'skip
': 'Requires vk account credentials
',
647 'url
': 'https
://vk
.com
/wall
-23538238_35',
648 'only_matching
': True,
650 # mobile wall page URL
651 'url
': 'https
://m
.vk
.com
/wall
-23538238_35',
652 'only_matching
': True,
654 _BASE64_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN0PQRSTUVWXYZO123456789
+/='
655 _AUDIO = collections.namedtuple('Audio
', ['id', 'owner_id
', 'url
', 'title
', 'performer
', 'duration
', 'album_id
', 'unk
', 'author_link
', 'lyrics
', 'flags
', 'context
', 'extra
', 'hashes
', 'cover_url
', 'ads
'])
657 def _decode(self, enc):
661 r = self._BASE64_CHARS.index(c)
663 e = 64 * e + r if cond else r
666 dec += chr(255 & e >> (-2 * n & 6))
669 def _unmask_url(self, mask_url, vk_id):
670 if 'audio_api_unavailable
' in mask_url:
671 extra = mask_url.split('?extra
=')[1].split('#')
672 func
, base
= self
._decode
(extra
[1]).split(chr(11))
673 mask_url
= list(self
._decode
(extra
[0]))
674 url_len
= len(mask_url
)
675 indexes
= [None] * url_len
676 index
= int(base
) ^ vk_id
677 for n
in range(url_len
- 1, -1, -1):
678 index
= (url_len
* (n
+ 1) ^ index
+ n
) % url_len
680 for n
in range(1, url_len
):
682 index
= indexes
[url_len
- 1 - n
]
683 mask_url
[n
] = mask_url
[index
]
685 mask_url
= ''.join(mask_url
)
688 def _real_extract(self
, url
):
689 post_id
= self
._match
_id
(url
)
691 webpage
= self
._download
_payload
('wkview', post_id
, {
693 'w': 'wall' + post_id
,
696 description
= clean_html(get_element_by_class('wall_post_text', webpage
))
697 uploader
= clean_html(get_element_by_class('author', webpage
))
701 for audio
in re
.findall(r
'data-audio="([^"]+)', webpage
):
702 audio
= self
._parse
_json
(unescapeHTML(audio
), post_id
)
703 a
= self
._AUDIO
._make
(audio
[:16])
706 title
= unescapeHTML(a
.title
)
707 performer
= unescapeHTML(a
.performer
)
709 'id': '%s_%s' % (a
.owner_id
, a
.id),
710 'url': self
._unmask
_url
(a
.url
, a
.ads
['vk_id']),
711 'title': '%s - %s' % (performer
, title
) if performer
else title
,
712 'thumbnails': [{'url': c_url}
for c_url
in a
.cover_url
.split(',')] if a
.cover_url
else None,
713 'duration': int_or_none(a
.duration
),
714 'uploader': uploader
,
718 'protocol': 'm3u8_native',
721 for video
in re
.finditer(
722 r
'<a[^>]+href=(["\'])(?P
<url
>/video(?
:-?
[\d_
]+).*?
)\
1', webpage):
723 entries.append(self.url_result(
724 compat_urlparse.urljoin(url, video.group('url
')), VKIE.ie_key()))
726 title = 'Wall post
%s' % post_id
728 return self.playlist_result(
729 orderedSet(entries), post_id,
730 '%s - %s' % (uploader, title) if uploader else title,