yt_dlp/extractor/yandexvideo.py

   1 import itertools
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     determine_ext,
   6     extract_attributes,
   7     int_or_none,
   8     lowercase_escape,
   9     parse_qs,
  10     traverse_obj,
  11     try_get,
  12     url_or_none,
  13 )
  14
  15
  16 class YandexVideoIE(InfoExtractor):
  17     _VALID_URL = r'''(?x)
  18                     https?://
  19                         (?:
  20                             yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
  21                             frontend\.vh\.yandex\.ru/player/
  22                         )
  23                         (?P<id>(?:[\da-f]{32}|[\w-]{12}))
  24                     '''
  25     _TESTS = [{
  26         'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
  27         'info_dict': {
  28             'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
  29             'ext': 'mp4',
  30             'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
  31             'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
  32             'thumbnail': r're:^https?://',
  33             'timestamp': 1549972939,
  34             'duration': 5575,
  35             'age_limit': 18,
  36             'upload_date': '20190212',
  37             'view_count': int,
  38             'like_count': int,
  39             'dislike_count': int,
  40         },
  41         'params': {'skip_download': 'm3u8'},
  42     }, {
  43         'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
  44         'only_matching': True,
  45     }, {
  46         'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
  47         'only_matching': True,
  48     }, {
  49         'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
  50         'only_matching': True,
  51     }, {
  52         # vod-episode, series episode
  53         'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
  54         'only_matching': True,
  55     }, {
  56         # episode, sports
  57         'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
  58         'only_matching': True,
  59     }, {
  60         # DASH with DRM
  61         'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
  62         'only_matching': True,
  63     }, {
  64         'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
  65         'only_matching': True,
  66     }]
  67
  68     def _real_extract(self, url):
  69         video_id = self._match_id(url)
  70
  71         player = try_get((self._download_json(
  72             'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
  73   player(content_id: "%s") {
  74     computed_title
  75     content_url
  76     description
  77     dislikes
  78     duration
  79     likes
  80     program_title
  81     release_date
  82     release_date_ut
  83     release_year
  84     restriction_age
  85     season
  86     start_time
  87     streams
  88     thumbnail
  89     title
  90     views_count
  91   }
  92 }''' % video_id).encode(), fatal=False)), lambda x: x['player']['content'])  # noqa: UP031
  93         if not player or player.get('error'):
  94             player = self._download_json(
  95                 f'https://frontend.vh.yandex.ru/v23/player/{video_id}.json',
  96                 video_id, query={
  97                     'stream_options': 'hires',
  98                     'disable_trackings': 1,
  99                 })
 100         content = player['content']
 101
 102         title = content.get('title') or content['computed_title']
 103
 104         formats = []
 105         streams = content.get('streams') or []
 106         streams.append({'url': content.get('content_url')})
 107         for stream in streams:
 108             content_url = url_or_none(stream.get('url'))
 109             if not content_url:
 110                 continue
 111             ext = determine_ext(content_url)
 112             if ext == 'ismc':
 113                 continue
 114             elif ext == 'm3u8':
 115                 formats.extend(self._extract_m3u8_formats(
 116                     content_url, video_id, 'mp4',
 117                     'm3u8_native', m3u8_id='hls', fatal=False))
 118             elif ext == 'mpd':
 119                 formats.extend(self._extract_mpd_formats(
 120                     content_url, video_id, mpd_id='dash', fatal=False))
 121             else:
 122                 formats.append({'url': content_url})
 123
 124         timestamp = (int_or_none(content.get('release_date'))
 125                      or int_or_none(content.get('release_date_ut'))
 126                      or int_or_none(content.get('start_time')))
 127         season = content.get('season') or {}
 128
 129         return {
 130             'id': video_id,
 131             'title': title,
 132             'description': content.get('description'),
 133             'thumbnail': content.get('thumbnail'),
 134             'timestamp': timestamp,
 135             'duration': int_or_none(content.get('duration')),
 136             'series': content.get('program_title'),
 137             'age_limit': int_or_none(content.get('restriction_age')),
 138             'view_count': int_or_none(content.get('views_count')),
 139             'like_count': int_or_none(content.get('likes')),
 140             'dislike_count': int_or_none(content.get('dislikes')),
 141             'season_number': int_or_none(season.get('season_number')),
 142             'season_id': season.get('id'),
 143             'release_year': int_or_none(content.get('release_year')),
 144             'formats': formats,
 145         }
 146
 147
 148 class YandexVideoPreviewIE(InfoExtractor):
 149     _VALID_URL = r'https?://(?:www\.)?yandex\.\w{2,3}(?:\.(?:am|ge|il|tr))?/video/preview(?:/?\?.*?filmId=|/)(?P<id>\d+)'
 150     _TESTS = [{  # Odnoklassniki
 151         'url': 'https://yandex.ru/video/preview/?filmId=10682852472978372885&text=summer',
 152         'info_dict': {
 153             'id': '1352565459459',
 154             'ext': 'mp4',
 155             'like_count': int,
 156             'upload_date': '20191202',
 157             'age_limit': 0,
 158             'duration': 196,
 159             'thumbnail': 'https://i.mycdn.me/videoPreview?id=544866765315&type=37&idx=13&tkn=TY5qjLYZHxpmcnK8U2LgzYkgmaU&fn=external_8',
 160             'uploader_id': '481054701571',
 161             'title': 'LOFT - summer, summer, summer HD',
 162             'uploader': 'АРТЁМ КУДРОВ',
 163         },
 164     }, {  # youtube
 165         'url': 'https://yandex.ru/video/preview/?filmId=4479424425337895262&source=main_redirect&text=видео&utm_source=main_stripe_big',
 166         'only_matching': True,
 167     }, {  # YandexVideo
 168         'url': 'https://yandex.ru/video/preview/5275069442094787341',
 169         'only_matching': True,
 170     }, {  # youtube
 171         'url': 'https://yandex.ru/video/preview/?filmId=16658118429797832897&from=tabbar&p=1&text=%D0%BF%D1%80%D0%BE%D1%81%D0%BC%D0%BE%D1%82%D1%80+%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82%D0%B0+%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9+%D0%BF%D1%80%D0%B8%D0%BD%D1%86+%D0%BC%D1%8B+%D0%B2+%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B5+%D0%B7%D0%B0+%D1%82%D0%B5%D1%85+%D0%BA%D0%BE%D0%B3%D0%BE+%D0%BF%D1%80%D0%B8%D1%80%D1%83%D1%87%D0%B8%D0%BB%D0%B8',
 172         'only_matching': True,
 173     }, {  # Odnoklassniki
 174         'url': 'https://yandex.ru/video/preview/?text=Francis%20Lai%20-%20Le%20Bon%20Et%20Les%20MC)chants&path=wizard&parent-reqid=1643208087979310-1481782809207673478-sas3-0931-2f9-sas-l7-balancer-8080-BAL-9380&wiz_type=vital&filmId=12508152936505397283',
 175         'only_matching': True,
 176     }, {  # Odnoklassniki
 177         'url': 'https://yandex.com/video/preview/?text=dossier%2051%20film%201978&path=yandex_search&parent-reqid=1664361087754492-8727541069609384458-sas2-0340-sas-l7-balancer-8080-BAL-8045&noreask=1&from_type=vast&filmId=5794987234584444632',
 178         'only_matching': True,
 179     }]
 180
 181     def _real_extract(self, url):
 182         video_id = self._match_id(url)
 183         webpage = self._download_webpage(url, video_id)
 184         data_raw = self._search_regex(r'window.Ya.__inline_params__\s*=\s*JSON.parse\(\'([^"]+?\\u0022video\\u0022:[^"]+?})\'\);', webpage, 'data_raw')
 185         data_json = self._parse_json(data_raw, video_id, transform_source=lowercase_escape)
 186         return self.url_result(data_json['video']['url'])
 187
 188
 189 class ZenYandexIE(InfoExtractor):
 190     _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
 191     _TESTS = [{
 192         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
 193         'info_dict': {
 194             'id': '60c7c443da18892ebfe85ed7',
 195             'ext': 'mp4',
 196             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
 197             'description': 'md5:8684912f6086f298f8078d4af0e8a600',
 198             'thumbnail': 're:^https://avatars.dzeninfra.ru/',
 199             'uploader': 'AcademeG DailyStream',
 200         },
 201         'params': {
 202             'skip_download': 'm3u8',
 203             'format': 'bestvideo',
 204         },
 205         'skip': 'The page does not exist',
 206     }, {
 207         'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
 208         'info_dict': {
 209             'id': '60c7c443da18892ebfe85ed7',
 210             'ext': 'mp4',
 211             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
 212             'description': 'md5:8684912f6086f298f8078d4af0e8a600',
 213             'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
 214             'uploader': 'AcademeG DailyStream',
 215             'upload_date': '20191111',
 216             'timestamp': 1573465585,
 217         },
 218         'params': {'skip_download': 'm3u8'},
 219     }, {
 220         'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
 221         'info_dict': {
 222             'id': '6002240ff8b1af50bb2da5e3',
 223             'ext': 'mp4',
 224             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 225             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 226             'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
 227             'uploader': 'TechInsider',
 228             'timestamp': 1611378221,
 229             'upload_date': '20210123',
 230         },
 231         'params': {'skip_download': 'm3u8'},
 232     }, {
 233         'url': 'https://dzen.ru/video/watch/6002240ff8b1af50bb2da5e3',
 234         'info_dict': {
 235             'id': '6002240ff8b1af50bb2da5e3',
 236             'ext': 'mp4',
 237             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 238             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 239             'thumbnail': 're:^https://avatars.dzeninfra.ru/',
 240             'uploader': 'TechInsider',
 241             'upload_date': '20210123',
 242             'timestamp': 1611378221,
 243         },
 244         'params': {'skip_download': 'm3u8'},
 245     }, {
 246         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
 247         'only_matching': True,
 248     }, {
 249         'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
 250         'only_matching': True,
 251     }]
 252
 253     def _real_extract(self, url):
 254         video_id = self._match_id(url)
 255         webpage = self._download_webpage(url, video_id)
 256         redirect = self._search_json(r'var it\s*=', webpage, 'redirect', id, default={}).get('retpath')
 257         if redirect:
 258             video_id = self._match_id(redirect)
 259             webpage = self._download_webpage(redirect, video_id, note='Redirecting')
 260         data_json = self._search_json(
 261             r'("data"\s*:|data\s*=)', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
 262         serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)', webpage, 'server state')
 263         uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
 264                                       webpage, 'uploader', default='<a>')
 265         uploader_name = extract_attributes(uploader).get('aria-label')
 266         item_id = traverse_obj(data_json, (serverstate, 'videoViewer', 'openedItemId', {str}))
 267         video_json = traverse_obj(data_json, (serverstate, 'videoViewer', 'items', item_id, {dict})) or {}
 268
 269         formats, subtitles = [], {}
 270         for s_url in traverse_obj(video_json, ('video', 'streams', ..., {url_or_none})):
 271             ext = determine_ext(s_url)
 272             if ext == 'mpd':
 273                 fmts, subs = self._extract_mpd_formats_and_subtitles(s_url, video_id, mpd_id='dash')
 274             elif ext == 'm3u8':
 275                 fmts, subs = self._extract_m3u8_formats_and_subtitles(s_url, video_id, 'mp4')
 276             formats.extend(fmts)
 277             subtitles = self._merge_subtitles(subtitles, subs)
 278         return {
 279             'id': video_id,
 280             'title': video_json.get('title') or self._og_search_title(webpage),
 281             'formats': formats,
 282             'subtitles': subtitles,
 283             'duration': int_or_none(video_json.get('duration')),
 284             'view_count': int_or_none(video_json.get('views')),
 285             'timestamp': int_or_none(video_json.get('publicationDate')),
 286             'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
 287             'description': video_json.get('description') or self._og_search_description(webpage),
 288             'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
 289         }
 290
 291
 292 class ZenYandexChannelIE(InfoExtractor):
 293     _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
 294     _TESTS = [{
 295         'url': 'https://zen.yandex.ru/tok_media',
 296         'info_dict': {
 297             'id': 'tok_media',
 298             'title': 'СПЕКТР',
 299             'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
 300         },
 301         'playlist_mincount': 169,
 302         'skip': 'The page does not exist',
 303     }, {
 304         'url': 'https://dzen.ru/tok_media',
 305         'info_dict': {
 306             'id': 'tok_media',
 307             'title': 'СПЕКТР',
 308             'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
 309         },
 310         'playlist_mincount': 169,
 311         'skip': 'The page does not exist',
 312     }, {
 313         'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
 314         'info_dict': {
 315             'id': '606fd806cc13cb3c58c05cf5',
 316             'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
 317             'title': 'AcademeG DailyStream',
 318         },
 319         'playlist_mincount': 657,
 320     }, {
 321         # Test that the playlist extractor finishes extracting when the
 322         # channel has less than one page
 323         'url': 'https://zen.yandex.ru/jony_me',
 324         'info_dict': {
 325             'id': 'jony_me',
 326             'description': 'md5:ce0a5cad2752ab58701b5497835b2cc5',
 327             'title': 'JONY ',
 328         },
 329         'playlist_count': 18,
 330     }, {
 331         # Test that the playlist extractor finishes extracting when the
 332         # channel has more than one page of entries
 333         'url': 'https://zen.yandex.ru/tatyanareva',
 334         'info_dict': {
 335             'id': 'tatyanareva',
 336             'description': 'md5:40a1e51f174369ec3ba9d657734ac31f',
 337             'title': 'Татьяна Рева',
 338             'entries': 'maxcount:200',
 339         },
 340         'playlist_mincount': 46,
 341     }, {
 342         'url': 'https://dzen.ru/id/606fd806cc13cb3c58c05cf5',
 343         'info_dict': {
 344             'id': '606fd806cc13cb3c58c05cf5',
 345             'title': 'AcademeG DailyStream',
 346             'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
 347         },
 348         'playlist_mincount': 657,
 349     }]
 350
 351     def _entries(self, item_id, server_state_json, server_settings_json):
 352         items = (traverse_obj(server_state_json, ('feed', 'items', ...))
 353                  or traverse_obj(server_settings_json, ('exportData', 'items', ...)))
 354
 355         more = (traverse_obj(server_state_json, ('links', 'more'))
 356                 or traverse_obj(server_settings_json, ('exportData', 'more', 'link')))
 357
 358         next_page_id = None
 359         for page in itertools.count(1):
 360             for item in items or []:
 361                 if item.get('type') != 'gif':
 362                     continue
 363                 video_id = traverse_obj(item, 'publication_id', 'publicationId') or ''
 364                 yield self.url_result(item['link'], ZenYandexIE, video_id.split(':')[-1])
 365
 366             current_page_id = next_page_id
 367             next_page_id = traverse_obj(parse_qs(more), ('next_page_id', -1))
 368             if not all((more, items, next_page_id, next_page_id != current_page_id)):
 369                 break
 370
 371             data = self._download_json(more, item_id, note=f'Downloading Page {page}')
 372             items, more = data.get('items'), traverse_obj(data, ('more', 'link'))
 373
 374     def _real_extract(self, url):
 375         item_id = self._match_id(url)
 376         webpage = self._download_webpage(url, item_id)
 377         redirect = self._search_json(
 378             r'var it\s*=', webpage, 'redirect', item_id, default={}).get('retpath')
 379         if redirect:
 380             item_id = self._match_id(redirect)
 381             webpage = self._download_webpage(redirect, item_id, note='Redirecting')
 382         data = self._search_json(
 383             r'("data"\s*:|data\s*=)', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
 384         server_state_json = traverse_obj(data, lambda k, _: k.startswith('__serverState__'), get_all=False)
 385         server_settings_json = traverse_obj(data, lambda k, _: k.startswith('__serverSettings__'), get_all=False)
 386
 387         return self.playlist_result(
 388             self._entries(item_id, server_state_json, server_settings_json),
 389             item_id, traverse_obj(server_state_json, ('channel', 'source', 'title')),
 390             traverse_obj(server_state_json, ('channel', 'source', 'description')))