yt_dlp/extractor/yandexvideo.py

   1 import itertools
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     determine_ext,
   6     extract_attributes,
   7     int_or_none,
   8     lowercase_escape,
   9     parse_qs,
  10     traverse_obj,
  11     try_get,
  12     url_or_none,
  13 )
  14
  15
  16 class YandexVideoIE(InfoExtractor):
  17     _VALID_URL = r'''(?x)
  18                     https?://
  19                         (?:
  20                             yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
  21                             frontend\.vh\.yandex\.ru/player/
  22                         )
  23                         (?P<id>(?:[\da-f]{32}|[\w-]{12}))
  24                     '''
  25     _TESTS = [{
  26         'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
  27         'info_dict': {
  28             'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
  29             'ext': 'mp4',
  30             'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
  31             'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
  32             'thumbnail': r're:^https?://',
  33             'timestamp': 1549972939,
  34             'duration': 5575,
  35             'age_limit': 18,
  36             'upload_date': '20190212',
  37             'view_count': int,
  38             'like_count': int,
  39             'dislike_count': int,
  40         },
  41         'params': {'skip_download': 'm3u8'},
  42     }, {
  43         'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
  44         'only_matching': True,
  45     }, {
  46         'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
  47         'only_matching': True,
  48     }, {
  49         'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
  50         'only_matching': True,
  51     }, {
  52         # vod-episode, series episode
  53         'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
  54         'only_matching': True,
  55     }, {
  56         # episode, sports
  57         'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
  58         'only_matching': True,
  59     }, {
  60         # DASH with DRM
  61         'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
  62         'only_matching': True,
  63     }, {
  64         'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
  65         'only_matching': True,
  66     }]
  67
  68     def _real_extract(self, url):
  69         video_id = self._match_id(url)
  70
  71         player = try_get((self._download_json(
  72             'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
  73   player(content_id: "%s") {
  74     computed_title
  75     content_url
  76     description
  77     dislikes
  78     duration
  79     likes
  80     program_title
  81     release_date
  82     release_date_ut
  83     release_year
  84     restriction_age
  85     season
  86     start_time
  87     streams
  88     thumbnail
  89     title
  90     views_count
  91   }
  92 }''' % video_id).encode(), fatal=False)), lambda x: x['player']['content'])
  93         if not player or player.get('error'):
  94             player = self._download_json(
  95                 'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,
  96                 video_id, query={
  97                     'stream_options': 'hires',
  98                     'disable_trackings': 1,
  99                 })
 100         content = player['content']
 101
 102         title = content.get('title') or content['computed_title']
 103
 104         formats = []
 105         streams = content.get('streams') or []
 106         streams.append({'url': content.get('content_url')})
 107         for stream in streams:
 108             content_url = url_or_none(stream.get('url'))
 109             if not content_url:
 110                 continue
 111             ext = determine_ext(content_url)
 112             if ext == 'ismc':
 113                 continue
 114             elif ext == 'm3u8':
 115                 formats.extend(self._extract_m3u8_formats(
 116                     content_url, video_id, 'mp4',
 117                     'm3u8_native', m3u8_id='hls', fatal=False))
 118             elif ext == 'mpd':
 119                 formats.extend(self._extract_mpd_formats(
 120                     content_url, video_id, mpd_id='dash', fatal=False))
 121             else:
 122                 formats.append({'url': content_url})
 123
 124         timestamp = (int_or_none(content.get('release_date'))
 125                      or int_or_none(content.get('release_date_ut'))
 126                      or int_or_none(content.get('start_time')))
 127         season = content.get('season') or {}
 128
 129         return {
 130             'id': video_id,
 131             'title': title,
 132             'description': content.get('description'),
 133             'thumbnail': content.get('thumbnail'),
 134             'timestamp': timestamp,
 135             'duration': int_or_none(content.get('duration')),
 136             'series': content.get('program_title'),
 137             'age_limit': int_or_none(content.get('restriction_age')),
 138             'view_count': int_or_none(content.get('views_count')),
 139             'like_count': int_or_none(content.get('likes')),
 140             'dislike_count': int_or_none(content.get('dislikes')),
 141             'season_number': int_or_none(season.get('season_number')),
 142             'season_id': season.get('id'),
 143             'release_year': int_or_none(content.get('release_year')),
 144             'formats': formats,
 145         }
 146
 147
 148 class YandexVideoPreviewIE(InfoExtractor):
 149     _VALID_URL = r'https?://(?:www\.)?yandex\.\w{2,3}(?:\.(?:am|ge|il|tr))?/video/preview(?:/?\?.*?filmId=|/)(?P<id>\d+)'
 150     _TESTS = [{  # Odnoklassniki
 151         'url': 'https://yandex.ru/video/preview/?filmId=10682852472978372885&text=summer',
 152         'info_dict': {
 153             'id': '1352565459459',
 154             'ext': 'mp4',
 155             'like_count': int,
 156             'upload_date': '20191202',
 157             'age_limit': 0,
 158             'duration': 196,
 159             'thumbnail': 'https://i.mycdn.me/videoPreview?id=544866765315&type=37&idx=13&tkn=TY5qjLYZHxpmcnK8U2LgzYkgmaU&fn=external_8',
 160             'uploader_id': '481054701571',
 161             'title': 'LOFT - summer, summer, summer HD',
 162             'uploader': 'АРТЁМ КУДРОВ',
 163         },
 164     }, {  # youtube
 165         'url': 'https://yandex.ru/video/preview/?filmId=4479424425337895262&source=main_redirect&text=видео&utm_source=main_stripe_big',
 166         'only_matching': True,
 167     }, {  # YandexVideo
 168         'url': 'https://yandex.ru/video/preview/5275069442094787341',
 169         'only_matching': True,
 170     }, {  # youtube
 171         'url': 'https://yandex.ru/video/preview/?filmId=16658118429797832897&from=tabbar&p=1&text=%D0%BF%D1%80%D0%BE%D1%81%D0%BC%D0%BE%D1%82%D1%80+%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82%D0%B0+%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9+%D0%BF%D1%80%D0%B8%D0%BD%D1%86+%D0%BC%D1%8B+%D0%B2+%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B5+%D0%B7%D0%B0+%D1%82%D0%B5%D1%85+%D0%BA%D0%BE%D0%B3%D0%BE+%D0%BF%D1%80%D0%B8%D1%80%D1%83%D1%87%D0%B8%D0%BB%D0%B8',
 172         'only_matching': True,
 173     }, {  # Odnoklassniki
 174         'url': 'https://yandex.ru/video/preview/?text=Francis%20Lai%20-%20Le%20Bon%20Et%20Les%20MC)chants&path=wizard&parent-reqid=1643208087979310-1481782809207673478-sas3-0931-2f9-sas-l7-balancer-8080-BAL-9380&wiz_type=vital&filmId=12508152936505397283',
 175         'only_matching': True,
 176     }, {  # Odnoklassniki
 177         'url': 'https://yandex.com/video/preview/?text=dossier%2051%20film%201978&path=yandex_search&parent-reqid=1664361087754492-8727541069609384458-sas2-0340-sas-l7-balancer-8080-BAL-8045&noreask=1&from_type=vast&filmId=5794987234584444632',
 178         'only_matching': True,
 179     }]
 180
 181     def _real_extract(self, url):
 182         id = self._match_id(url)
 183         webpage = self._download_webpage(url, id)
 184         data_raw = self._search_regex(r'window.Ya.__inline_params__\s*=\s*JSON.parse\(\'([^"]+?\\u0022video\\u0022:[^"]+?})\'\);', webpage, 'data_raw')
 185         data_json = self._parse_json(data_raw, id, transform_source=lowercase_escape)
 186         return self.url_result(data_json['video']['url'])
 187
 188
 189 class ZenYandexIE(InfoExtractor):
 190     _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
 191     _TESTS = [{
 192         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
 193         'info_dict': {
 194             'id': '60c7c443da18892ebfe85ed7',
 195             'ext': 'mp4',
 196             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
 197             'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
 198             'thumbnail': 're:^https://avatars.dzeninfra.ru/',
 199             'uploader': 'AcademeG DailyStream'
 200         },
 201         'params': {
 202             'skip_download': 'm3u8',
 203             'format': 'bestvideo',
 204         },
 205         'skip': 'The page does not exist',
 206     }, {
 207         'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
 208         'info_dict': {
 209             'id': '60c7c443da18892ebfe85ed7',
 210             'ext': 'mp4',
 211             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
 212             'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
 213             'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
 214             'uploader': 'AcademeG DailyStream',
 215             'upload_date': '20191111',
 216             'timestamp': 1573465585,
 217         },
 218         'params': {'skip_download': 'm3u8'},
 219     }, {
 220         'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
 221         'info_dict': {
 222             'id': '6002240ff8b1af50bb2da5e3',
 223             'ext': 'mp4',
 224             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 225             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 226             'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
 227             'uploader': 'TechInsider',
 228             'timestamp': 1611378221,
 229             'upload_date': '20210123',
 230         },
 231         'params': {'skip_download': 'm3u8'},
 232     }, {
 233         'url': 'https://dzen.ru/video/watch/6002240ff8b1af50bb2da5e3',
 234         'info_dict': {
 235             'id': '6002240ff8b1af50bb2da5e3',
 236             'ext': 'mp4',
 237             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 238             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 239             'thumbnail': 're:^https://avatars.dzeninfra.ru/',
 240             'uploader': 'TechInsider',
 241             'upload_date': '20210123',
 242             'timestamp': 1611378221,
 243         },
 244         'params': {'skip_download': 'm3u8'},
 245     }, {
 246         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
 247         'only_matching': True,
 248     }, {
 249         'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
 250         'only_matching': True,
 251     }]
 252
 253     def _real_extract(self, url):
 254         video_id = self._match_id(url)
 255         webpage = self._download_webpage(url, video_id)
 256         redirect = self._search_json(r'var it\s*=', webpage, 'redirect', id, default={}).get('retpath')
 257         if redirect:
 258             video_id = self._match_id(redirect)
 259             webpage = self._download_webpage(redirect, video_id, note='Redirecting')
 260         data_json = self._search_json(
 261             r'data\s*=', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
 262         serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
 263                                          webpage, 'server state').replace('State', 'Settings')
 264         uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
 265                                       webpage, 'uploader', default='<a>')
 266         uploader_name = extract_attributes(uploader).get('aria-label')
 267         video_json = try_get(data_json, lambda x: x[serverstate]['exportData']['video'], dict)
 268         stream_urls = try_get(video_json, lambda x: x['video']['streams'])
 269         formats = []
 270         for s_url in stream_urls:
 271             ext = determine_ext(s_url)
 272             if ext == 'mpd':
 273                 formats.extend(self._extract_mpd_formats(s_url, id, mpd_id='dash'))
 274             elif ext == 'm3u8':
 275                 formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4'))
 276         return {
 277             'id': video_id,
 278             'title': video_json.get('title') or self._og_search_title(webpage),
 279             'formats': formats,
 280             'duration': int_or_none(video_json.get('duration')),
 281             'view_count': int_or_none(video_json.get('views')),
 282             'timestamp': int_or_none(video_json.get('publicationDate')),
 283             'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
 284             'description': self._og_search_description(webpage) or try_get(data_json, lambda x: x['og']['description']),
 285             'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
 286         }
 287
 288
 289 class ZenYandexChannelIE(InfoExtractor):
 290     _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
 291     _TESTS = [{
 292         'url': 'https://zen.yandex.ru/tok_media',
 293         'info_dict': {
 294             'id': 'tok_media',
 295             'title': 'СПЕКТР',
 296             'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
 297         },
 298         'playlist_mincount': 169,
 299     }, {
 300         'url': 'https://dzen.ru/tok_media',
 301         'info_dict': {
 302             'id': 'tok_media',
 303             'title': 'СПЕКТР',
 304             'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
 305         },
 306         'playlist_mincount': 169,
 307     }, {
 308         'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
 309         'info_dict': {
 310             'id': '606fd806cc13cb3c58c05cf5',
 311             'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
 312             'title': 'AcademeG DailyStream',
 313         },
 314         'playlist_mincount': 657,
 315     }, {
 316         # Test that the playlist extractor finishes extracting when the
 317         # channel has less than one page
 318         'url': 'https://zen.yandex.ru/jony_me',
 319         'info_dict': {
 320             'id': 'jony_me',
 321             'description': 'md5:a2c62b4ef5cf3e3efb13d25f61f739e1',
 322             'title': 'JONY ',
 323         },
 324         'playlist_count': 20,
 325     }, {
 326         # Test that the playlist extractor finishes extracting when the
 327         # channel has more than one page of entries
 328         'url': 'https://zen.yandex.ru/tatyanareva',
 329         'info_dict': {
 330             'id': 'tatyanareva',
 331             'description': 'md5:296b588d60841c3756c9105f237b70c6',
 332             'title': 'Татьяна Рева',
 333             'entries': 'maxcount:200',
 334         },
 335         'playlist_count': 46,
 336     }, {
 337         'url': 'https://dzen.ru/id/606fd806cc13cb3c58c05cf5',
 338         'info_dict': {
 339             'id': '606fd806cc13cb3c58c05cf5',
 340             'title': 'AcademeG DailyStream',
 341             'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
 342         },
 343         'playlist_mincount': 657,
 344     }]
 345
 346     def _entries(self, item_id, server_state_json, server_settings_json):
 347         items = (traverse_obj(server_state_json, ('feed', 'items', ...))
 348                  or traverse_obj(server_settings_json, ('exportData', 'items', ...)))
 349
 350         more = (traverse_obj(server_state_json, ('links', 'more'))
 351                 or traverse_obj(server_settings_json, ('exportData', 'more', 'link')))
 352
 353         next_page_id = None
 354         for page in itertools.count(1):
 355             for item in items or []:
 356                 if item.get('type') != 'gif':
 357                     continue
 358                 video_id = traverse_obj(item, 'publication_id', 'publicationId') or ''
 359                 yield self.url_result(item['link'], ZenYandexIE, video_id.split(':')[-1])
 360
 361             current_page_id = next_page_id
 362             next_page_id = traverse_obj(parse_qs(more), ('next_page_id', -1))
 363             if not all((more, items, next_page_id, next_page_id != current_page_id)):
 364                 break
 365
 366             data = self._download_json(more, item_id, note=f'Downloading Page {page}')
 367             items, more = data.get('items'), traverse_obj(data, ('more', 'link'))
 368
 369     def _real_extract(self, url):
 370         item_id = self._match_id(url)
 371         webpage = self._download_webpage(url, item_id)
 372         redirect = self._search_json(
 373             r'var it\s*=', webpage, 'redirect', item_id, default={}).get('retpath')
 374         if redirect:
 375             item_id = self._match_id(redirect)
 376             webpage = self._download_webpage(redirect, item_id, note='Redirecting')
 377         data = self._search_json(
 378             r'var\s+data\s*=', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
 379         server_state_json = traverse_obj(data, lambda k, _: k.startswith('__serverState__'), get_all=False)
 380         server_settings_json = traverse_obj(data, lambda k, _: k.startswith('__serverSettings__'), get_all=False)
 381
 382         return self.playlist_result(
 383             self._entries(item_id, server_state_json, server_settings_json),
 384             item_id, traverse_obj(server_state_json, ('channel', 'source', 'title')),
 385             traverse_obj(server_state_json, ('channel', 'source', 'description')))