yt_dlp/extractor/yandexvideo.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import itertools
   5 import re
   6
   7 from .common import InfoExtractor
   8 from ..utils import (
   9     determine_ext,
  10     extract_attributes,
  11     int_or_none,
  12     try_get,
  13     url_or_none,
  14 )
  15
  16
  17 class YandexVideoIE(InfoExtractor):
  18     _VALID_URL = r'''(?x)
  19                     https?://
  20                         (?:
  21                             yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
  22                             frontend\.vh\.yandex\.ru/player/
  23                         )
  24                         (?P<id>(?:[\da-f]{32}|[\w-]{12}))
  25                     '''
  26     _TESTS = [{
  27         'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
  28         'md5': 'e02a05bfaf0d9615ef07ae3a10f4faf4',
  29         'info_dict': {
  30             'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
  31             'ext': 'mp4',
  32             'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
  33             'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
  34             'thumbnail': r're:^https?://',
  35             'timestamp': 1549972939,
  36             'duration': 5575,
  37             'age_limit': 18,
  38             'upload_date': '20190212',
  39             'view_count': int,
  40             'like_count': int,
  41             'dislike_count': int,
  42         },
  43     }, {
  44         'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
  45         'only_matching': True,
  46     }, {
  47         'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
  48         'only_matching': True,
  49     }, {
  50         'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
  51         'only_matching': True,
  52     }, {
  53         # vod-episode, series episode
  54         'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
  55         'only_matching': True,
  56     }, {
  57         # episode, sports
  58         'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
  59         'only_matching': True,
  60     }, {
  61         # DASH with DRM
  62         'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
  63         'only_matching': True,
  64     }, {
  65         'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
  66         'only_matching': True,
  67     }]
  68
  69     def _real_extract(self, url):
  70         video_id = self._match_id(url)
  71
  72         player = try_get((self._download_json(
  73             'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
  74   player(content_id: "%s") {
  75     computed_title
  76     content_url
  77     description
  78     dislikes
  79     duration
  80     likes
  81     program_title
  82     release_date
  83     release_date_ut
  84     release_year
  85     restriction_age
  86     season
  87     start_time
  88     streams
  89     thumbnail
  90     title
  91     views_count
  92   }
  93 }''' % video_id).encode(), fatal=False)), lambda x: x['player']['content'])
  94         if not player or player.get('error'):
  95             player = self._download_json(
  96                 'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,
  97                 video_id, query={
  98                     'stream_options': 'hires',
  99                     'disable_trackings': 1,
 100                 })
 101         content = player['content']
 102
 103         title = content.get('title') or content['computed_title']
 104
 105         formats = []
 106         streams = content.get('streams') or []
 107         streams.append({'url': content.get('content_url')})
 108         for stream in streams:
 109             content_url = url_or_none(stream.get('url'))
 110             if not content_url:
 111                 continue
 112             ext = determine_ext(content_url)
 113             if ext == 'ismc':
 114                 continue
 115             elif ext == 'm3u8':
 116                 formats.extend(self._extract_m3u8_formats(
 117                     content_url, video_id, 'mp4',
 118                     'm3u8_native', m3u8_id='hls', fatal=False))
 119             elif ext == 'mpd':
 120                 formats.extend(self._extract_mpd_formats(
 121                     content_url, video_id, mpd_id='dash', fatal=False))
 122             else:
 123                 formats.append({'url': content_url})
 124
 125         self._sort_formats(formats)
 126
 127         timestamp = (int_or_none(content.get('release_date'))
 128                      or int_or_none(content.get('release_date_ut'))
 129                      or int_or_none(content.get('start_time')))
 130         season = content.get('season') or {}
 131
 132         return {
 133             'id': video_id,
 134             'title': title,
 135             'description': content.get('description'),
 136             'thumbnail': content.get('thumbnail'),
 137             'timestamp': timestamp,
 138             'duration': int_or_none(content.get('duration')),
 139             'series': content.get('program_title'),
 140             'age_limit': int_or_none(content.get('restriction_age')),
 141             'view_count': int_or_none(content.get('views_count')),
 142             'like_count': int_or_none(content.get('likes')),
 143             'dislike_count': int_or_none(content.get('dislikes')),
 144             'season_number': int_or_none(season.get('season_number')),
 145             'season_id': season.get('id'),
 146             'release_year': int_or_none(content.get('release_year')),
 147             'formats': formats,
 148         }
 149
 150
 151 class ZenYandexIE(InfoExtractor):
 152     _VALID_URL = r'https?://zen\.yandex\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
 153     _TESTS = [{
 154         'url': 'https://zen.yandex.ru/media/popmech/izverjenie-vulkana-iz-spichek-zreliscnyi-opyt-6002240ff8b1af50bb2da5e3',
 155         'info_dict': {
 156             'id': '6002240ff8b1af50bb2da5e3',
 157             'ext': 'mp4',
 158             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 159             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 160             'thumbnail': 're:^https://avatars.mds.yandex.net/',
 161             'uploader': 'Популярная механика',
 162         },
 163         'params': {
 164             'skip_download': 'm3u8',
 165         },
 166     }, {
 167         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
 168         'info_dict': {
 169             'id': '60c7c443da18892ebfe85ed7',
 170             'ext': 'mp4',
 171             'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
 172             'description': 'md5:f3db3d995763b9bbb7b56d4ccdedea89',
 173             'thumbnail': 're:^https://avatars.mds.yandex.net/',
 174             'uploader': 'AcademeG DailyStream'
 175         },
 176         'params': {
 177             'skip_download': 'm3u8',
 178             'format': 'bestvideo',
 179         },
 180     }, {
 181         'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
 182         'info_dict': {
 183             'id': '6002240ff8b1af50bb2da5e3',
 184             'ext': 'mp4',
 185             'title': 'Извержение вулкана из спичек: зрелищный опыт',
 186             'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
 187             'uploader': 'Популярная механика',
 188         },
 189         'params': {
 190             'skip_download': 'm3u8',
 191         },
 192     }, {
 193         'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
 194         'only_matching': True,
 195     }]
 196
 197     def _real_extract(self, url):
 198         id = self._match_id(url)
 199         webpage = self._download_webpage(url, id)
 200         data_json = self._parse_json(
 201             self._search_regex(r'data\s*=\s*({["\']_*serverState_*video.+?});', webpage, 'metadata'), id)
 202         serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)',
 203                                          webpage, 'server state').replace('State', 'Settings')
 204         uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
 205                                       webpage, 'uploader', default='<a>')
 206         uploader_name = extract_attributes(uploader).get('aria-label')
 207         video_json = try_get(data_json, lambda x: x[serverstate]['exportData']['video'], dict)
 208         stream_urls = try_get(video_json, lambda x: x['video']['streams'])
 209         formats = []
 210         for s_url in stream_urls:
 211             ext = determine_ext(s_url)
 212             if ext == 'mpd':
 213                 formats.extend(self._extract_mpd_formats(s_url, id, mpd_id='dash'))
 214             elif ext == 'm3u8':
 215                 formats.extend(self._extract_m3u8_formats(s_url, id, 'mp4'))
 216         self._sort_formats(formats)
 217         return {
 218             'id': id,
 219             'title': video_json.get('title') or self._og_search_title(webpage),
 220             'formats': formats,
 221             'duration': int_or_none(video_json.get('duration')),
 222             'view_count': int_or_none(video_json.get('views')),
 223             'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
 224             'description': self._og_search_description(webpage) or try_get(data_json, lambda x: x['og']['description']),
 225             'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
 226         }
 227
 228
 229 class ZenYandexChannelIE(InfoExtractor):
 230     _VALID_URL = r'https?://zen\.yandex\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
 231     _TESTS = [{
 232         'url': 'https://zen.yandex.ru/tok_media',
 233         'info_dict': {
 234             'id': 'tok_media',
 235         },
 236         'playlist_mincount': 169,
 237     }, {
 238         'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
 239         'info_dict': {
 240             'id': '606fd806cc13cb3c58c05cf5',
 241         },
 242         'playlist_mincount': 657,
 243     }]
 244
 245     def _entries(self, id, url):
 246         webpage = self._download_webpage(url, id)
 247         data_json = self._parse_json(re.findall(r'var\s?data\s?=\s?({.+?})\s?;', webpage)[-1], id)
 248         for key in data_json.keys():
 249             if key.startswith('__serverState__'):
 250                 data_json = data_json[key]
 251         items = list(try_get(data_json, lambda x: x['feed']['items'], dict).values())
 252         more = try_get(data_json, lambda x: x['links']['more']) or None
 253         for page in itertools.count(1):
 254             for item in items:
 255                 video_id = item.get('publication_id') or item.get('publicationId')
 256                 video_url = item.get('link')
 257                 yield self.url_result(video_url, ie=ZenYandexIE.ie_key(), video_id=video_id.split(':')[-1])
 258             if not more:
 259                 break
 260             data_json = self._download_json(more, id, note='Downloading Page %d' % page)
 261             items = data_json.get('items', [])
 262             more = try_get(data_json, lambda x: x['more']['link']) or None
 263
 264     def _real_extract(self, url):
 265         id = self._match_id(url)
 266         return self.playlist_result(self._entries(id, url), playlist_id=id)