yt_dlp/extractor/yandexvideo.py

   1 import itertools
   2 import re
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     determine_ext,
   7     extract_attributes,
   8     int_or_none,
   9     try_get,
  10     url_or_none,
  11     lowercase_escape,
  12 )
  13
  14
  15 class YandexVideoIE(InfoExtractor):
  16     _VALID_URL = r'''(?x)
  17                     https?://
  18                         (?:
  19                             yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
  20                             frontend\.vh\.yandex\.ru/player/
  21                         )
  22                         (?P<id>(?:[\da-f]{32}|[\w-]{12}))
  23                     '''
  24     _TESTS = [{
  25         'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
  26         'md5': 'e02a05bfaf0d9615ef07ae3a10f4faf4',
  27         'info_dict': {
  28             'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
  29             'ext': 'mp4',
  30             'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
  31             'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
  32             'thumbnail': r're:^https?://',
  33             'timestamp': 1549972939,
  34             'duration': 5575,
  35             'age_limit': 18,
  36             'upload_date': '20190212',
  37             'view_count': int,
  38             'like_count': int,
  39             'dislike_count': int,
  40         },
  41     }, {
  42         'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
  43         'only_matching': True,
  44     }, {
  45         'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
  46         'only_matching': True,
  47     }, {
  48         'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
  49         'only_matching': True,
  50     }, {
  51         # vod-episode, series episode
  52         'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
  53         'only_matching': True,
  54     }, {
  55         # episode, sports
  56         'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
  57         'only_matching': True,
  58     }, {
  59         # DASH with DRM
  60         'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
  61         'only_matching': True,
  62     }, {
  63         'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
  64         'only_matching': True,
  65     }]
  66
  67     def _real_extract(self, url):
  68         video_id = self._match_id(url)
  69
  70         player = try_get((self._download_json(
  71             'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
  72   player(content_id: "%s") {
  73     computed_title
  74     content_url
  75     description
  76     dislikes
  77     duration
  78     likes
  79     program_title
  80     release_date
  81     release_date_ut
  82     release_year
  83     restriction_age
  84     season
  85     start_time
  86     streams
  87     thumbnail
  88     title
  89     views_count
  90   }
  91 }''' % video_id).encode(), fatal=False)), lambda x: x['player']['content'])
  92         if not player or player.get('error'):
  93             player = self._download_json(
  94                 'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,
  95                 video_id, query={
  96                     'stream_options': 'hires',
  97                     'disable_trackings': 1,
  98                 })
  99         content = player['content']
 100
 101         title = content.get('title') or content['computed_title']
 102
 103         formats = []
 104         streams = content.get('streams') or []
 105         streams.append({'url': content.get('content_url')})
 106         for stream in streams:
 107             content_url = url_or_none(stream.get('url'))
 108             if not content_url:
 109                 continue
 110             ext = determine_ext(content_url)
 111             if ext == 'ismc':
 112                 continue
 113             elif ext == 'm3u8':
 114                 formats.extend(self._extract_m3u8_formats(
 115                     content_url, video_id, 'mp4',
 116                     'm3u8_native', m3u8_id='hls', fatal=False))
 117             elif ext == 'mpd':
 118                 formats.extend(self._extract_mpd_formats(
 119                     content_url, video_id, mpd_id='dash', fatal=False))
 120             else:
 121                 formats.append({'url': content_url})
 122
 123         self._sort_formats(formats)
 124
 125         timestamp = (int_or_none(content.get('release_date'))
 126                      or int_or_none(content.get('release_date_ut'))
 127                      or int_or_none(content.get('start_time')))
 128         season = content.get('season') or {}
 129
 130         return {
 131             'id': video_id,
 132             'title': title,
 133             'description': content.get('description'),
 134             'thumbnail': content.get('thumbnail'),
 135             'timestamp': timestamp,
 136             'duration': int_or_none(content.get('duration')),
 137             'series': content.get('program_title'),
 138             'age_limit': int_or_none(content.get('restriction_age')),
 139             'view_count': int_or_none(content.get('views_count')),
 140             'like_count': int_or_none(content.get('likes')),
 141             'dislike_count': int_or_none(content.get('dislikes')),
 142             'season_number': int_or_none(season.get('season_number')),
 143             'season_id': season.get('id'),
 144             'release_year': int_or_none(content.get('release_year')),
 145             'formats': formats,
 146         }
 147
 148
 149 class YandexVideoPreviewIE(InfoExtractor):
 150     _VALID_URL = r'https?://(?:www\.)?yandex\.ru/video/preview(?:/?\?.*?filmId=|/)(?P<id>\d+)'
 151     _TESTS = [{  # Odnoklassniki
 152         'url': 'https://yandex.ru/video/preview/?filmId=10682852472978372885&text=summer',
 153         'info_dict': {
 154             'id': '1352565459459',
 155             'ext': 'mp4',
 156             'like_count': int,
 157             'upload_date': '20191202',
 158             'age_limit': 0,
 159             'duration': 196,
 160             'thumbnail': 'https://i.mycdn.me/videoPreview?id=544866765315&type=37&idx=13&tkn=TY5qjLYZHxpmcnK8U2LgzYkgmaU&fn=external_8',
 161             'uploader_id': '481054701571',
 162             'title': 'LOFT - summer, summer, summer HD',
 163             'uploader': 'АРТЁМ КУДРОВ',
 164         },
 165     }, {  # youtube
 166         'url': 'https://yandex.ru/video/preview/?filmId=4479424425337895262&source=main_redirect&text=видео&utm_source=main_stripe_big',
 167         'only_matching': True,
 168     }, {  # YandexVideo
 169         'url': 'https://yandex.ru/video/preview/5275069442094787341',
 170         'only_matching': True,
 171     }, {  # youtube
 172         'url': 'https://yandex.ru/video/preview/?filmId=16658118429797832897&from=tabbar&p=1&text=%D0%BF%D1%80%D0%BE%D1%81%D0%BC%D0%BE%D1%82%D1%80+%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82%D0%B0+%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9+%D0%BF%D1%80%D0%B8%D0%BD%D1%86+%D0%BC%D1%8B+%D0%B2+%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B5+%D0%B7%D0%B0+%D1%82%D0%B5%D1%85+%D0%BA%D0%BE%D0%B3%D0%BE+%D0%BF%D1%80%D0%B8%D1%80%D1%83%D1%87%D0%B8%D0%BB%D0%B8',
 173         'only_matching': True,
 174     }, {  # Odnoklassniki
 175         'url': 'https://yandex.ru/video/preview/?text=Francis%20Lai%20-%20Le%20Bon%20Et%20Les%20MC)chants&path=wizard&parent-reqid=1643208087979310-1481782809207673478-sas3-0931-2f9-sas-l7-balancer-8080-BAL-9380&wiz_type=vital&filmId=12508152936505397283',
 176         'only_matching': True,
 177     }]
 178
 179     def _real_extract(self, url):
 180         id = self._match_id(url)
 181         webpage = self._download_webpage(url, id)
 182         data_raw = self._search_regex(r'window.Ya.__inline_params__\s*=\s*JSON.parse\(\'([^"]+?\\u0022video\\u0022:[^"]+?})\'\);', webpage, 'data_raw')
 183         data_json = self._parse_json(data_raw, id, transform_source=lowercase_escape)
 184         return self.url_result(data_json['video']['url'])
 185
 186
 187 class ZenYandexIE(InfoExtractor):
 188     _VALID_URL = r'https?://zen\.yandex\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
 189     _TESTS = [{
 190         'url': 'https://zen.yandex.ru/media/popmech/izverjenie-vulkana-iz-spichek-zreliscnyi-opyt-6002240ff8b1af50bb2da5e3',
 191         'info_dict': {
 192             'id': '6002240ff8b1af50bb2da5e3',
 193             'ext': 'mp4',
 194             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 195             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 196             'thumbnail': 're:^https://avatars.mds.yandex.net/',
 197             'uploader': 'Популярная механика',
 198         },
 199         'params': {
 200             'skip_download': 'm3u8',
 201         },
 202     }, {
 203         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
 204         'info_dict': {
 205             'id': '60c7c443da18892ebfe85ed7',
 206             'ext': 'mp4',
 207             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
 208             'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
 209             'thumbnail': 're:^https://avatars.mds.yandex.net/',
 210             'uploader': 'AcademeG DailyStream'
 211         },
 212         'params': {
 213             'skip_download': 'm3u8',
 214             'format': 'bestvideo',
 215         },
 216     }, {
 217         'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
 218         'info_dict': {
 219             'id': '6002240ff8b1af50bb2da5e3',
 220             'ext': 'mp4',
 221             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 222             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 223             'uploader': 'Популярная механика',
 224         },
 225         'params': {
 226             'skip_download': 'm3u8',
 227         },
 228     }, {
 229         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
 230         'only_matching': True,
 231     }]
 232
 233     def _real_extract(self, url):
 234         id = self._match_id(url)
 235         webpage = self._download_webpage(url, id)
 236         data_json = self._parse_json(
 237             self._search_regex(r'data\s*=\s*({["\']_*serverState_*video.+?});', webpage, 'metadata'), id)
 238         serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
 239                                          webpage, 'server state').replace('State', 'Settings')
 240         uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
 241                                       webpage, 'uploader', default='<a>')
 242         uploader_name = extract_attributes(uploader).get('aria-label')
 243         video_json = try_get(data_json, lambda x: x[serverstate]['exportData']['video'], dict)
 244         stream_urls = try_get(video_json, lambda x: x['video']['streams'])
 245         formats = []
 246         for s_url in stream_urls:
 247             ext = determine_ext(s_url)
 248             if ext == 'mpd':
 249                 formats.extend(self._extract_mpd_formats(s_url, id, mpd_id='dash'))
 250             elif ext == 'm3u8':
 251                 formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4'))
 252         self._sort_formats(formats)
 253         return {
 254             'id': id,
 255             'title': video_json.get('title') or self._og_search_title(webpage),
 256             'formats': formats,
 257             'duration': int_or_none(video_json.get('duration')),
 258             'view_count': int_or_none(video_json.get('views')),
 259             'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
 260             'description': self._og_search_description(webpage) or try_get(data_json, lambda x: x['og']['description']),
 261             'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
 262         }
 263
 264
 265 class ZenYandexChannelIE(InfoExtractor):
 266     _VALID_URL = r'https?://zen\.yandex\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
 267     _TESTS = [{
 268         'url': 'https://zen.yandex.ru/tok_media',
 269         'info_dict': {
 270             'id': 'tok_media',
 271         },
 272         'playlist_mincount': 169,
 273     }, {
 274         'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
 275         'info_dict': {
 276             'id': '606fd806cc13cb3c58c05cf5',
 277         },
 278         'playlist_mincount': 657,
 279     }]
 280
 281     def _entries(self, id, url):
 282         webpage = self._download_webpage(url, id)
 283         data_json = self._parse_json(re.findall(r'var\s?data\s?=\s?({.+?})\s?;', webpage)[-1], id)
 284         for key in data_json.keys():
 285             if key.startswith('__serverState__'):
 286                 data_json = data_json[key]
 287         items = list(try_get(data_json, lambda x: x['feed']['items'], dict).values())
 288         more = try_get(data_json, lambda x: x['links']['more']) or None
 289         for page in itertools.count(1):
 290             for item in items:
 291                 video_id = item.get('publication_id') or item.get('publicationId')
 292                 video_url = item.get('link')
 293                 yield self.url_result(video_url, ie=ZenYandexIE.ie_key(), video_id=video_id.split(':')[-1])
 294             if not more:
 295                 break
 296             data_json = self._download_json(more, id, note='Downloading Page %d' % page)
 297             items = data_json.get('items', [])
 298             more = try_get(data_json, lambda x: x['more']['link']) or None
 299
 300     def _real_extract(self, url):
 301         id = self._match_id(url)
 302         return self.playlist_result(self._entries(id, url), playlist_id=id)