[yt-dlp.git] / yt_dlp / extractor / yandexvideo.py

import itertools

from .common import InfoExtractor
from ..utils import (
    determine_ext,
    extract_attributes,
    int_or_none,
    lowercase_escape,
    parse_qs,
    traverse_obj,
    try_get,
    url_or_none,
)


class YandexVideoIE(InfoExtractor):
    _VALID_URL = r'''(?x)
                    https?://
                        (?:
                            yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
                            frontend\.vh\.yandex\.ru/player/
                        )
                        (?P<id>(?:[\da-f]{32}|[\w-]{12}))
                    '''
    _TESTS = [{
        'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
        'info_dict': {
            'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
            'ext': 'mp4',
            'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
            'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
            'thumbnail': r're:^https?://',
            'timestamp': 1549972939,
            'duration': 5575,
            'age_limit': 18,
            'upload_date': '20190212',
            'view_count': int,
            'like_count': int,
            'dislike_count': int,
        },
        'params': {'skip_download': 'm3u8'},
    }, {
        'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
        'only_matching': True,
    }, {
        'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
        'only_matching': True,
    }, {
        'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
        'only_matching': True,
    }, {
        # vod-episode, series episode
        'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
        'only_matching': True,
    }, {
        # episode, sports
        'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
        'only_matching': True,
    }, {
        # DASH with DRM
        'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
        'only_matching': True,
    }, {
        'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        player = try_get((self._download_json(
            'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
  player(content_id: "%s") {
    computed_title
    content_url
    description
    dislikes
    duration
    likes
    program_title
    release_date
    release_date_ut
    release_year
    restriction_age
    season
    start_time
    streams
    thumbnail
    title
    views_count
  }
}''' % video_id).encode(), fatal=False)), lambda x: x['player']['content'])  # noqa: UP031
        if not player or player.get('error'):
            player = self._download_json(
                f'https://frontend.vh.yandex.ru/v23/player/{video_id}.json',
                video_id, query={
                    'stream_options': 'hires',
                    'disable_trackings': 1,
                })
        content = player['content']

        title = content.get('title') or content['computed_title']

        formats = []
        streams = content.get('streams') or []
        streams.append({'url': content.get('content_url')})
        for stream in streams:
            content_url = url_or_none(stream.get('url'))
            if not content_url:
                continue
            ext = determine_ext(content_url)
            if ext == 'ismc':
                continue
            elif ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    content_url, video_id, 'mp4',
                    'm3u8_native', m3u8_id='hls', fatal=False))
            elif ext == 'mpd':
                formats.extend(self._extract_mpd_formats(
                    content_url, video_id, mpd_id='dash', fatal=False))
            else:
                formats.append({'url': content_url})

        timestamp = (int_or_none(content.get('release_date'))
                     or int_or_none(content.get('release_date_ut'))
                     or int_or_none(content.get('start_time')))
        season = content.get('season') or {}

        return {
            'id': video_id,
            'title': title,
            'description': content.get('description'),
            'thumbnail': content.get('thumbnail'),
            'timestamp': timestamp,
            'duration': int_or_none(content.get('duration')),
            'series': content.get('program_title'),
            'age_limit': int_or_none(content.get('restriction_age')),
            'view_count': int_or_none(content.get('views_count')),
            'like_count': int_or_none(content.get('likes')),
            'dislike_count': int_or_none(content.get('dislikes')),
            'season_number': int_or_none(season.get('season_number')),
            'season_id': season.get('id'),
            'release_year': int_or_none(content.get('release_year')),
            'formats': formats,
        }


class YandexVideoPreviewIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?yandex\.\w{2,3}(?:\.(?:am|ge|il|tr))?/video/preview(?:/?\?.*?filmId=|/)(?P<id>\d+)'
    _TESTS = [{  # Odnoklassniki
        'url': 'https://yandex.ru/video/preview/?filmId=10682852472978372885&text=summer',
        'info_dict': {
            'id': '1352565459459',
            'ext': 'mp4',
            'like_count': int,
            'upload_date': '20191202',
            'age_limit': 0,
            'duration': 196,
            'thumbnail': 'https://i.mycdn.me/videoPreview?id=544866765315&type=37&idx=13&tkn=TY5qjLYZHxpmcnK8U2LgzYkgmaU&fn=external_8',
            'uploader_id': '481054701571',
            'title': 'LOFT - summer, summer, summer HD',
            'uploader': 'АРТЁМ КУДРОВ',
        },
    }, {  # youtube
        'url': 'https://yandex.ru/video/preview/?filmId=4479424425337895262&source=main_redirect&text=видео&utm_source=main_stripe_big',
        'only_matching': True,
    }, {  # YandexVideo
        'url': 'https://yandex.ru/video/preview/5275069442094787341',
        'only_matching': True,
    }, {  # youtube
        'url': 'https://yandex.ru/video/preview/?filmId=16658118429797832897&from=tabbar&p=1&text=%D0%BF%D1%80%D0%BE%D1%81%D0%BC%D0%BE%D1%82%D1%80+%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82%D0%B0+%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9+%D0%BF%D1%80%D0%B8%D0%BD%D1%86+%D0%BC%D1%8B+%D0%B2+%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B5+%D0%B7%D0%B0+%D1%82%D0%B5%D1%85+%D0%BA%D0%BE%D0%B3%D0%BE+%D0%BF%D1%80%D0%B8%D1%80%D1%83%D1%87%D0%B8%D0%BB%D0%B8',
        'only_matching': True,
    }, {  # Odnoklassniki
        'url': 'https://yandex.ru/video/preview/?text=Francis%20Lai%20-%20Le%20Bon%20Et%20Les%20MC)chants&path=wizard&parent-reqid=1643208087979310-1481782809207673478-sas3-0931-2f9-sas-l7-balancer-8080-BAL-9380&wiz_type=vital&filmId=12508152936505397283',
        'only_matching': True,
    }, {  # Odnoklassniki
        'url': 'https://yandex.com/video/preview/?text=dossier%2051%20film%201978&path=yandex_search&parent-reqid=1664361087754492-8727541069609384458-sas2-0340-sas-l7-balancer-8080-BAL-8045&noreask=1&from_type=vast&filmId=5794987234584444632',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        data_raw = self._search_regex(r'window.Ya.__inline_params__\s*=\s*JSON.parse\(\'([^"]+?\\u0022video\\u0022:[^"]+?})\'\);', webpage, 'data_raw')
        data_json = self._parse_json(data_raw, video_id, transform_source=lowercase_escape)
        return self.url_result(data_json['video']['url'])


class ZenYandexIE(InfoExtractor):
    _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru(?:/video)?/(media|watch)/(?:(?:id/[^/]+/|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
    _TESTS = [{
        'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
        'info_dict': {
            'id': '60c7c443da18892ebfe85ed7',
            'ext': 'mp4',
            'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
            'description': 'md5:8684912f6086f298f8078d4af0e8a600',
            'thumbnail': 're:^https://avatars.dzeninfra.ru/',
            'uploader': 'AcademeG DailyStream',
        },
        'params': {
            'skip_download': 'm3u8',
            'format': 'bestvideo',
        },
        'skip': 'The page does not exist',
    }, {
        'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
        'info_dict': {
            'id': '60c7c443da18892ebfe85ed7',
            'ext': 'mp4',
            'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
            'description': 'md5:8684912f6086f298f8078d4af0e8a600',
            'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
            'uploader': 'AcademeG DailyStream',
            'upload_date': '20191111',
            'timestamp': 1573465585,
        },
        'params': {'skip_download': 'm3u8'},
    }, {
        'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
        'info_dict': {
            'id': '6002240ff8b1af50bb2da5e3',
            'ext': 'mp4',
            'title': 'Извержение вулкана из спичек: зрелищный опыт',
            'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
            'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
            'uploader': 'TechInsider',
            'timestamp': 1611378221,
            'upload_date': '20210123',
        },
        'params': {'skip_download': 'm3u8'},
    }, {
        'url': 'https://dzen.ru/video/watch/6002240ff8b1af50bb2da5e3',
        'info_dict': {
            'id': '6002240ff8b1af50bb2da5e3',
            'ext': 'mp4',
            'title': 'Извержение вулкана из спичек: зрелищный опыт',
            'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
            'thumbnail': 're:^https://avatars.dzeninfra.ru/',
            'uploader': 'TechInsider',
            'upload_date': '20210123',
            'timestamp': 1611378221,
        },
        'params': {'skip_download': 'm3u8'},
    }, {
        'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
        'only_matching': True,
    }, {
        'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)
        redirect = self._search_json(r'var it\s*=', webpage, 'redirect', id, default={}).get('retpath')
        if redirect:
            video_id = self._match_id(redirect)
            webpage = self._download_webpage(redirect, video_id, note='Redirecting')
        data_json = self._search_json(
            r'("data"\s*:|data\s*=)', webpage, 'metadata', video_id, contains_pattern=r'{["\']_*serverState_*video.+}')
        serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)', webpage, 'server state')
        uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
                                      webpage, 'uploader', default='<a>')
        uploader_name = extract_attributes(uploader).get('aria-label')
        item_id = traverse_obj(data_json, (serverstate, 'videoViewer', 'openedItemId', {str}))
        video_json = traverse_obj(data_json, (serverstate, 'videoViewer', 'items', item_id, {dict})) or {}

        formats, subtitles = [], {}
        for s_url in traverse_obj(video_json, ('video', 'streams', ..., {url_or_none})):
            ext = determine_ext(s_url)
            if ext == 'mpd':
                fmts, subs = self._extract_mpd_formats_and_subtitles(s_url, video_id, mpd_id='dash')
            elif ext == 'm3u8':
                fmts, subs = self._extract_m3u8_formats_and_subtitles(s_url, video_id, 'mp4')
            formats.extend(fmts)
            subtitles = self._merge_subtitles(subtitles, subs)
        return {
            'id': video_id,
            'title': video_json.get('title') or self._og_search_title(webpage),
            'formats': formats,
            'subtitles': subtitles,
            'duration': int_or_none(video_json.get('duration')),
            'view_count': int_or_none(video_json.get('views')),
            'timestamp': int_or_none(video_json.get('publicationDate')),
            'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
            'description': video_json.get('description') or self._og_search_description(webpage),
            'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
        }


class ZenYandexChannelIE(InfoExtractor):
    _VALID_URL = r'https?://(zen\.yandex|dzen)\.ru/(?!media|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
    _TESTS = [{
        'url': 'https://zen.yandex.ru/tok_media',
        'info_dict': {
            'id': 'tok_media',
            'title': 'СПЕКТР',
            'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
        },
        'playlist_mincount': 169,
        'skip': 'The page does not exist',
    }, {
        'url': 'https://dzen.ru/tok_media',
        'info_dict': {
            'id': 'tok_media',
            'title': 'СПЕКТР',
            'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
        },
        'playlist_mincount': 169,
        'skip': 'The page does not exist',
    }, {
        'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
        'info_dict': {
            'id': '606fd806cc13cb3c58c05cf5',
            'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
            'title': 'AcademeG DailyStream',
        },
        'playlist_mincount': 657,
    }, {
        # Test that the playlist extractor finishes extracting when the
        # channel has less than one page
        'url': 'https://zen.yandex.ru/jony_me',
        'info_dict': {
            'id': 'jony_me',
            'description': 'md5:ce0a5cad2752ab58701b5497835b2cc5',
            'title': 'JONY ',
        },
        'playlist_count': 18,
    }, {
        # Test that the playlist extractor finishes extracting when the
        # channel has more than one page of entries
        'url': 'https://zen.yandex.ru/tatyanareva',
        'info_dict': {
            'id': 'tatyanareva',
            'description': 'md5:40a1e51f174369ec3ba9d657734ac31f',
            'title': 'Татьяна Рева',
            'entries': 'maxcount:200',
        },
        'playlist_mincount': 46,
    }, {
        'url': 'https://dzen.ru/id/606fd806cc13cb3c58c05cf5',
        'info_dict': {
            'id': '606fd806cc13cb3c58c05cf5',
            'title': 'AcademeG DailyStream',
            'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
        },
        'playlist_mincount': 657,
    }]

    def _entries(self, item_id, server_state_json, server_settings_json):
        items = (traverse_obj(server_state_json, ('feed', 'items', ...))
                 or traverse_obj(server_settings_json, ('exportData', 'items', ...)))

        more = (traverse_obj(server_state_json, ('links', 'more'))
                or traverse_obj(server_settings_json, ('exportData', 'more', 'link')))

        next_page_id = None
        for page in itertools.count(1):
            for item in items or []:
                if item.get('type') != 'gif':
                    continue
                video_id = traverse_obj(item, 'publication_id', 'publicationId') or ''
                yield self.url_result(item['link'], ZenYandexIE, video_id.split(':')[-1])

            current_page_id = next_page_id
            next_page_id = traverse_obj(parse_qs(more), ('next_page_id', -1))
            if not all((more, items, next_page_id, next_page_id != current_page_id)):
                break

            data = self._download_json(more, item_id, note=f'Downloading Page {page}')
            items, more = data.get('items'), traverse_obj(data, ('more', 'link'))

    def _real_extract(self, url):
        item_id = self._match_id(url)
        webpage = self._download_webpage(url, item_id)
        redirect = self._search_json(
            r'var it\s*=', webpage, 'redirect', item_id, default={}).get('retpath')
        if redirect:
            item_id = self._match_id(redirect)
            webpage = self._download_webpage(redirect, item_id, note='Redirecting')
        data = self._search_json(
            r'("data"\s*:|data\s*=)', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
        server_state_json = traverse_obj(data, lambda k, _: k.startswith('__serverState__'), get_all=False)
        server_settings_json = traverse_obj(data, lambda k, _: k.startswith('__serverSettings__'), get_all=False)

        return self.playlist_result(
            self._entries(item_id, server_state_json, server_settings_json),
            item_id, traverse_obj(server_state_json, ('channel', 'source', 'title')),
            traverse_obj(server_state_json, ('channel', 'source', 'description')))
Commit	Line	Data
dc9de9cb	1	import itertools
dc9de9cb	2
8428fdcc S	3	from .common import InfoExtractor
8428fdcc S	4	from ..utils import (
33b529fa	5	determine_ext,
a109acbf	6	extract_attributes,
8428fdcc	7	int_or_none,
304ad45a	8	lowercase_escape,
c9bd6518 AK	9	parse_qs,
c9bd6518 AK	10	traverse_obj,
29f7c58a	11	try_get,
8428fdcc S	12	url_or_none,
	13	)
	14
	15
	16	class YandexVideoIE(InfoExtractor):
	17	_VALID_URL = r'''(?x)
	18	https?://
	19	(?:
29f7c58a	20	yandex\.ru(?:/(?:portal/(?:video\|efir)\|efir))?/?\?.*?stream_id=\|
8428fdcc S	21	frontend\.vh\.yandex\.ru/player/
8428fdcc S	22	)
29f7c58a	23	(?P<id>(?:[\da-f]{32}\|[\w-]{12}))
8428fdcc S	24	'''
8428fdcc S	25	_TESTS = [{
29f7c58a	26	'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
8428fdcc	27	'info_dict': {
29f7c58a	28	'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
8428fdcc	29	'ext': 'mp4',
29f7c58a	30	'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
	31	'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
	32	'thumbnail': r're:^https?://',
	33	'timestamp': 1549972939,
	34	'duration': 5575,
8428fdcc	35	'age_limit': 18,
29f7c58a	36	'upload_date': '20190212',
	37	'view_count': int,
	38	'like_count': int,
	39	'dislike_count': int,
8428fdcc	40	},
c9bd6518	41	'params': {'skip_download': 'm3u8'},
8428fdcc	42	}, {
29f7c58a	43	'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
8428fdcc S	44	'only_matching': True,
	45	}, {
	46	'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
	47	'only_matching': True,
	48	}, {
	49	'url': 'https://frontend.vh.yandex.ru/player/4dbb262b4fe5cf15a215de4f34eee34d?from=morda',
	50	'only_matching': True,
	51	}, {
	52	# vod-episode, series episode
	53	'url': 'https://yandex.ru/portal/video?stream_id=45b11db6e4b68797919c93751a938cee',
	54	'only_matching': True,
	55	}, {
	56	# episode, sports
	57	'url': 'https://yandex.ru/?stream_channel=1538487871&stream_id=4132a07f71fb0396be93d74b3477131d',
	58	'only_matching': True,
33b529fa S	59	}, {
	60	# DASH with DRM
	61	'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
	62	'only_matching': True,
29f7c58a	63	}, {
	64	'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
	65	'only_matching': True,
8428fdcc S	66	}]
	67
	68	def _real_extract(self, url):
	69	video_id = self._match_id(url)
	70
29f7c58a	71	player = try_get((self._download_json(
	72	'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
	73	player(content_id: "%s") {
	74	computed_title
	75	content_url
	76	description
	77	dislikes
	78	duration
	79	likes
	80	program_title
	81	release_date
	82	release_date_ut
	83	release_year
	84	restriction_age
	85	season
	86	start_time
	87	streams
	88	thumbnail
	89	title
	90	views_count
	91	}
add96eb9	92	}''' % video_id).encode(), fatal=False)), lambda x: x['player']['content']) # noqa: UP031
29f7c58a	93	if not player or player.get('error'):
29f7c58a	94	player = self._download_json(
add96eb9	95	f'https://frontend.vh.yandex.ru/v23/player/{video_id}.json',
29f7c58a	96	video_id, query={
	97	'stream_options': 'hires',
	98	'disable_trackings': 1,
	99	})
	100	content = player['content']
8428fdcc	101
29f7c58a	102	title = content.get('title') or content['computed_title']
33b529fa	103
29f7c58a	104	formats = []
	105	streams = content.get('streams') or []
	106	streams.append({'url': content.get('content_url')})
	107	for stream in streams:
	108	content_url = url_or_none(stream.get('url'))
	109	if not content_url:
	110	continue
	111	ext = determine_ext(content_url)
	112	if ext == 'ismc':
	113	continue
	114	elif ext == 'm3u8':
	115	formats.extend(self._extract_m3u8_formats(
	116	content_url, video_id, 'mp4',
	117	'm3u8_native', m3u8_id='hls', fatal=False))
	118	elif ext == 'mpd':
	119	formats.extend(self._extract_mpd_formats(
	120	content_url, video_id, mpd_id='dash', fatal=False))
	121	else:
	122	formats.append({'url': content_url})
33b529fa	123
3089bc74 S	124	timestamp = (int_or_none(content.get('release_date'))
	125	or int_or_none(content.get('release_date_ut'))
	126	or int_or_none(content.get('start_time')))
29f7c58a	127	season = content.get('season') or {}
8428fdcc S	128
	129	return {
	130	'id': video_id,
	131	'title': title,
29f7c58a	132	'description': content.get('description'),
29f7c58a	133	'thumbnail': content.get('thumbnail'),
8428fdcc	134	'timestamp': timestamp,
29f7c58a	135	'duration': int_or_none(content.get('duration')),
	136	'series': content.get('program_title'),
	137	'age_limit': int_or_none(content.get('restriction_age')),
	138	'view_count': int_or_none(content.get('views_count')),
	139	'like_count': int_or_none(content.get('likes')),
	140	'dislike_count': int_or_none(content.get('dislikes')),
	141	'season_number': int_or_none(season.get('season_number')),
	142	'season_id': season.get('id'),
	143	'release_year': int_or_none(content.get('release_year')),
8428fdcc S	144	'formats': formats,
8428fdcc S	145	}
dc9de9cb A	146
dc9de9cb A	147
e26f9cc1	148	class YandexVideoPreviewIE(InfoExtractor):
2e0f8d4f	149	_VALID_URL = r'https?://(?:www\.)?yandex\.\w{2,3}(?:\.(?:am\|ge\|il\|tr))?/video/preview(?:/?\?.*?filmId=\|/)(?P<id>\d+)'
e26f9cc1 K	150	_TESTS = [{ # Odnoklassniki
	151	'url': 'https://yandex.ru/video/preview/?filmId=10682852472978372885&text=summer',
	152	'info_dict': {
	153	'id': '1352565459459',
	154	'ext': 'mp4',
	155	'like_count': int,
	156	'upload_date': '20191202',
	157	'age_limit': 0,
	158	'duration': 196,
	159	'thumbnail': 'https://i.mycdn.me/videoPreview?id=544866765315&type=37&idx=13&tkn=TY5qjLYZHxpmcnK8U2LgzYkgmaU&fn=external_8',
	160	'uploader_id': '481054701571',
	161	'title': 'LOFT - summer, summer, summer HD',
e26f9cc1 K	162	'uploader': 'АРТЁМ КУДРОВ',
	163	},
	164	}, { # youtube
	165	'url': 'https://yandex.ru/video/preview/?filmId=4479424425337895262&source=main_redirect&text=видео&utm_source=main_stripe_big',
	166	'only_matching': True,
	167	}, { # YandexVideo
	168	'url': 'https://yandex.ru/video/preview/5275069442094787341',
	169	'only_matching': True,
	170	}, { # youtube
	171	'url': 'https://yandex.ru/video/preview/?filmId=16658118429797832897&from=tabbar&p=1&text=%D0%BF%D1%80%D0%BE%D1%81%D0%BC%D0%BE%D1%82%D1%80+%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82%D0%B0+%D0%BC%D0%B0%D0%BB%D0%B5%D0%BD%D1%8C%D0%BA%D0%B8%D0%B9+%D0%BF%D1%80%D0%B8%D0%BD%D1%86+%D0%BC%D1%8B+%D0%B2+%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B5+%D0%B7%D0%B0+%D1%82%D0%B5%D1%85+%D0%BA%D0%BE%D0%B3%D0%BE+%D0%BF%D1%80%D0%B8%D1%80%D1%83%D1%87%D0%B8%D0%BB%D0%B8',
	172	'only_matching': True,
	173	}, { # Odnoklassniki
	174	'url': 'https://yandex.ru/video/preview/?text=Francis%20Lai%20-%20Le%20Bon%20Et%20Les%20MC)chants&path=wizard&parent-reqid=1643208087979310-1481782809207673478-sas3-0931-2f9-sas-l7-balancer-8080-BAL-9380&wiz_type=vital&filmId=12508152936505397283',
	175	'only_matching': True,
2e0f8d4f SS	176	}, { # Odnoklassniki
	177	'url': 'https://yandex.com/video/preview/?text=dossier%2051%20film%201978&path=yandex_search&parent-reqid=1664361087754492-8727541069609384458-sas2-0340-sas-l7-balancer-8080-BAL-8045&noreask=1&from_type=vast&filmId=5794987234584444632',
	178	'only_matching': True,
e26f9cc1 K	179	}]
	180
	181	def _real_extract(self, url):
add96eb9	182	video_id = self._match_id(url)
add96eb9	183	webpage = self._download_webpage(url, video_id)
e26f9cc1	184	data_raw = self._search_regex(r'window.Ya.__inline_params__\s=\sJSON.parse\(\'([^"]+?\\u0022video\\u0022:[^"]+?})\'\);', webpage, 'data_raw')
add96eb9	185	data_json = self._parse_json(data_raw, video_id, transform_source=lowercase_escape)
e26f9cc1 K	186	return self.url_result(data_json['video']['url'])
	187
	188
dc9de9cb	189	class ZenYandexIE(InfoExtractor):
c9bd6518	190	_VALID_URL = r'https?://(zen\.yandex\|dzen)\.ru(?:/video)?/(media\|watch)/(?:(?:id/[^/]+/\|[^/]+/)(?:[a-z0-9-]+)-)?(?P<id>[a-z0-9-]+)'
dc9de9cb	191	_TESTS = [{
c9bd6518	192	'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
dc9de9cb	193	'info_dict': {
c9bd6518	194	'id': '60c7c443da18892ebfe85ed7',
dc9de9cb	195	'ext': 'mp4',
c9bd6518	196	'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
5efe68b7	197	'description': 'md5:8684912f6086f298f8078d4af0e8a600',
c9bd6518	198	'thumbnail': 're:^https://avatars.dzeninfra.ru/',
add96eb9	199	'uploader': 'AcademeG DailyStream',
dc9de9cb	200	},
a109acbf	201	'params': {
a109acbf	202	'skip_download': 'm3u8',
c9bd6518	203	'format': 'bestvideo',
a109acbf	204	},
c9bd6518	205	'skip': 'The page does not exist',
dc9de9cb	206	}, {
c9bd6518	207	'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/vot-eto-focus-dedy-morozy-na-gidrociklah-60c7c443da18892ebfe85ed7',
dc9de9cb A	208	'info_dict': {
	209	'id': '60c7c443da18892ebfe85ed7',
	210	'ext': 'mp4',
	211	'title': 'ВОТ ЭТО Focus. Деды Морозы на гидроциклах',
5efe68b7	212	'description': 'md5:8684912f6086f298f8078d4af0e8a600',
c9bd6518 AK	213	'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
	214	'uploader': 'AcademeG DailyStream',
	215	'upload_date': '20191111',
	216	'timestamp': 1573465585,
a109acbf	217	},
c9bd6518	218	'params': {'skip_download': 'm3u8'},
a109acbf	219	}, {
	220	'url': 'https://zen.yandex.ru/video/watch/6002240ff8b1af50bb2da5e3',
	221	'info_dict': {
	222	'id': '6002240ff8b1af50bb2da5e3',
	223	'ext': 'mp4',
	224	'title': 'Извержение вулкана из спичек: зрелищный опыт',
	225	'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
c9bd6518 AK	226	'thumbnail': r're:^https://avatars\.dzeninfra\.ru/',
	227	'uploader': 'TechInsider',
	228	'timestamp': 1611378221,
	229	'upload_date': '20210123',
a109acbf	230	},
c9bd6518 AK	231	'params': {'skip_download': 'm3u8'},
	232	}, {
	233	'url': 'https://dzen.ru/video/watch/6002240ff8b1af50bb2da5e3',
	234	'info_dict': {
	235	'id': '6002240ff8b1af50bb2da5e3',
	236	'ext': 'mp4',
	237	'title': 'Извержение вулкана из спичек: зрелищный опыт',
	238	'description': 'md5:053ad3c61b5596d510c9a199dc8ee633',
	239	'thumbnail': 're:^https://avatars.dzeninfra.ru/',
	240	'uploader': 'TechInsider',
	241	'upload_date': '20210123',
	242	'timestamp': 1611378221,
a109acbf	243	},
c9bd6518	244	'params': {'skip_download': 'm3u8'},
dc9de9cb A	245	}, {
	246	'url': 'https://zen.yandex.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
	247	'only_matching': True,
c9bd6518 AK	248	}, {
	249	'url': 'https://dzen.ru/media/id/606fd806cc13cb3c58c05cf5/novyi-samsung-fold-3-moskvich-barahlit-612f93b7f8d48e7e945792a2?from=channel&rid=2286618386.482.1630817595976.42360',
	250	'only_matching': True,
dc9de9cb A	251	}]
	252
	253	def _real_extract(self, url):
c9bd6518 AK	254	video_id = self._match_id(url)
c9bd6518 AK	255	webpage = self._download_webpage(url, video_id)
46d09f87	256	redirect = self._search_json(r'var it\s*=', webpage, 'redirect', id, default={}).get('retpath')
c9bd6518 AK	257	if redirect:
	258	video_id = self._match_id(redirect)
	259	webpage = self._download_webpage(redirect, video_id, note='Redirecting')
	260	data_json = self._search_json(
5efe68b7	261	r'("data"\s:\|data\s=)', webpage, 'metadata', video_id, contains_pattern=r'{["\']_serverState_video.+}')
c4b87dd8	262	serverstate = self._search_regex(r'(_+serverState_+video-site_[^_]+_+)', webpage, 'server state')
a109acbf	263	uploader = self._search_regex(r'(<a\s*class=["\']card-channel-link[^"\']+["\'][^>]+>)',
	264	webpage, 'uploader', default='<a>')
	265	uploader_name = extract_attributes(uploader).get('aria-label')
c4b87dd8	266	item_id = traverse_obj(data_json, (serverstate, 'videoViewer', 'openedItemId', {str}))
	267	video_json = traverse_obj(data_json, (serverstate, 'videoViewer', 'items', item_id, {dict})) or {}
	268
5efe68b7	269	formats, subtitles = [], {}
c4b87dd8	270	for s_url in traverse_obj(video_json, ('video', 'streams', ..., {url_or_none})):
a109acbf	271	ext = determine_ext(s_url)
a109acbf	272	if ext == 'mpd':
5efe68b7	273	fmts, subs = self._extract_mpd_formats_and_subtitles(s_url, video_id, mpd_id='dash')
a109acbf	274	elif ext == 'm3u8':
5efe68b7 BN	275	fmts, subs = self._extract_m3u8_formats_and_subtitles(s_url, video_id, 'mp4')
	276	formats.extend(fmts)
	277	subtitles = self._merge_subtitles(subtitles, subs)
dc9de9cb	278	return {
c9bd6518	279	'id': video_id,
a109acbf	280	'title': video_json.get('title') or self._og_search_title(webpage),
dc9de9cb	281	'formats': formats,
5efe68b7	282	'subtitles': subtitles,
a109acbf	283	'duration': int_or_none(video_json.get('duration')),
a109acbf	284	'view_count': int_or_none(video_json.get('views')),
c9bd6518	285	'timestamp': int_or_none(video_json.get('publicationDate')),
a109acbf	286	'uploader': uploader_name or data_json.get('authorName') or try_get(data_json, lambda x: x['publisher']['name']),
5efe68b7	287	'description': video_json.get('description') or self._og_search_description(webpage),
a109acbf	288	'thumbnail': self._og_search_thumbnail(webpage) or try_get(data_json, lambda x: x['og']['imageUrl']),
dc9de9cb A	289	}
	290
	291
	292	class ZenYandexChannelIE(InfoExtractor):
c9bd6518	293	_VALID_URL = r'https?://(zen\.yandex\|dzen)\.ru/(?!media\|video)(?:id/)?(?P<id>[a-z0-9-_]+)'
dc9de9cb A	294	_TESTS = [{
	295	'url': 'https://zen.yandex.ru/tok_media',
	296	'info_dict': {
	297	'id': 'tok_media',
c9bd6518 AK	298	'title': 'СПЕКТР',
	299	'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
	300	},
	301	'playlist_mincount': 169,
5efe68b7	302	'skip': 'The page does not exist',
c9bd6518 AK	303	}, {
	304	'url': 'https://dzen.ru/tok_media',
	305	'info_dict': {
	306	'id': 'tok_media',
	307	'title': 'СПЕКТР',
	308	'description': 'md5:a9e5b3c247b7fe29fd21371a428bcf56',
dc9de9cb A	309	},
dc9de9cb A	310	'playlist_mincount': 169,
5efe68b7	311	'skip': 'The page does not exist',
dc9de9cb A	312	}, {
	313	'url': 'https://zen.yandex.ru/id/606fd806cc13cb3c58c05cf5',
	314	'info_dict': {
	315	'id': '606fd806cc13cb3c58c05cf5',
c9bd6518 AK	316	'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
	317	'title': 'AcademeG DailyStream',
	318	},
	319	'playlist_mincount': 657,
	320	}, {
	321	# Test that the playlist extractor finishes extracting when the
	322	# channel has less than one page
	323	'url': 'https://zen.yandex.ru/jony_me',
	324	'info_dict': {
	325	'id': 'jony_me',
5efe68b7	326	'description': 'md5:ce0a5cad2752ab58701b5497835b2cc5',
c9bd6518 AK	327	'title': 'JONY ',
c9bd6518 AK	328	},
5efe68b7	329	'playlist_count': 18,
c9bd6518 AK	330	}, {
	331	# Test that the playlist extractor finishes extracting when the
	332	# channel has more than one page of entries
	333	'url': 'https://zen.yandex.ru/tatyanareva',
	334	'info_dict': {
	335	'id': 'tatyanareva',
5efe68b7	336	'description': 'md5:40a1e51f174369ec3ba9d657734ac31f',
c9bd6518 AK	337	'title': 'Татьяна Рева',
	338	'entries': 'maxcount:200',
	339	},
5efe68b7	340	'playlist_mincount': 46,
c9bd6518 AK	341	}, {
	342	'url': 'https://dzen.ru/id/606fd806cc13cb3c58c05cf5',
	343	'info_dict': {
	344	'id': '606fd806cc13cb3c58c05cf5',
	345	'title': 'AcademeG DailyStream',
	346	'description': 'md5:517b7c97d8ca92e940f5af65448fd928',
dc9de9cb A	347	},
	348	'playlist_mincount': 657,
	349	}]
	350
c9bd6518 AK	351	def _entries(self, item_id, server_state_json, server_settings_json):
	352	items = (traverse_obj(server_state_json, ('feed', 'items', ...))
	353	or traverse_obj(server_settings_json, ('exportData', 'items', ...)))
	354
	355	more = (traverse_obj(server_state_json, ('links', 'more'))
	356	or traverse_obj(server_settings_json, ('exportData', 'more', 'link')))
	357
	358	next_page_id = None
dc9de9cb	359	for page in itertools.count(1):
c9bd6518 AK	360	for item in items or []:
	361	if item.get('type') != 'gif':
	362	continue
	363	video_id = traverse_obj(item, 'publication_id', 'publicationId') or ''
	364	yield self.url_result(item['link'], ZenYandexIE, video_id.split(':')[-1])
	365
	366	current_page_id = next_page_id
	367	next_page_id = traverse_obj(parse_qs(more), ('next_page_id', -1))
	368	if not all((more, items, next_page_id, next_page_id != current_page_id)):
dc9de9cb	369	break
c9bd6518 AK	370
	371	data = self._download_json(more, item_id, note=f'Downloading Page {page}')
	372	items, more = data.get('items'), traverse_obj(data, ('more', 'link'))
dc9de9cb A	373
dc9de9cb A	374	def _real_extract(self, url):
c9bd6518 AK	375	item_id = self._match_id(url)
	376	webpage = self._download_webpage(url, item_id)
	377	redirect = self._search_json(
46d09f87	378	r'var it\s*=', webpage, 'redirect', item_id, default={}).get('retpath')
c9bd6518 AK	379	if redirect:
	380	item_id = self._match_id(redirect)
	381	webpage = self._download_webpage(redirect, item_id, note='Redirecting')
	382	data = self._search_json(
5efe68b7	383	r'("data"\s:\|data\s=)', webpage, 'channel data', item_id, contains_pattern=r'{\"__serverState__.+}')
c9bd6518 AK	384	server_state_json = traverse_obj(data, lambda k, _: k.startswith('__serverState__'), get_all=False)
	385	server_settings_json = traverse_obj(data, lambda k, _: k.startswith('__serverSettings__'), get_all=False)
	386
	387	return self.playlist_result(
	388	self._entries(item_id, server_state_json, server_settings_json),
	389	item_id, traverse_obj(server_state_json, ('channel', 'source', 'title')),
	390	traverse_obj(server_state_json, ('channel', 'source', 'description')))