[yt-dlp.git] / yt_dlp / downloader / youtube_live_chat.py

from __future__ import division, unicode_literals

import json
import time

from .fragment import FragmentFD
from ..compat import compat_urllib_error
from ..utils import (
    try_get,
    dict_get,
    int_or_none,
    RegexNotFoundError,
)
from ..extractor.youtube import YoutubeBaseInfoExtractor as YT_BaseIE


class YoutubeLiveChatFD(FragmentFD):
    """ Downloads YouTube live chats fragment by fragment """

    FD_NAME = 'youtube_live_chat'

    def real_download(self, filename, info_dict):
        video_id = info_dict['video_id']
        self.to_screen('[%s] Downloading live chat' % self.FD_NAME)

        fragment_retries = self.params.get('fragment_retries', 0)
        test = self.params.get('test', False)

        ctx = {
            'filename': filename,
            'live': True,
            'total_frags': None,
        }

        ie = YT_BaseIE(self.ydl)

        start_time = int(time.time() * 1000)

        def dl_fragment(url, data=None, headers=None):
            http_headers = info_dict.get('http_headers', {})
            if headers:
                http_headers = http_headers.copy()
                http_headers.update(headers)
            return self._download_fragment(ctx, url, info_dict, http_headers, data)

        def parse_actions_replay(live_chat_continuation):
            offset = continuation_id = click_tracking_params = None
            processed_fragment = bytearray()
            for action in live_chat_continuation.get('actions', []):
                if 'replayChatItemAction' in action:
                    replay_chat_item_action = action['replayChatItemAction']
                    offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
                processed_fragment.extend(
                    json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
            if offset is not None:
                continuation = try_get(
                    live_chat_continuation,
                    lambda x: x['continuations'][0]['liveChatReplayContinuationData'], dict)
                if continuation:
                    continuation_id = continuation.get('continuation')
                    click_tracking_params = continuation.get('clickTrackingParams')
            self._append_fragment(ctx, processed_fragment)
            return continuation_id, offset, click_tracking_params

        def try_refresh_replay_beginning(live_chat_continuation):
            # choose the second option that contains the unfiltered live chat replay
            refresh_continuation = try_get(
                live_chat_continuation,
                lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData'], dict)
            if refresh_continuation:
                # no data yet but required to call _append_fragment
                self._append_fragment(ctx, b'')
                refresh_continuation_id = refresh_continuation.get('continuation')
                offset = 0
                click_tracking_params = refresh_continuation.get('trackingParams')
                return refresh_continuation_id, offset, click_tracking_params
            return parse_actions_replay(live_chat_continuation)

        live_offset = 0

        def parse_actions_live(live_chat_continuation):
            nonlocal live_offset
            continuation_id = click_tracking_params = None
            processed_fragment = bytearray()
            for action in live_chat_continuation.get('actions', []):
                timestamp = self.parse_live_timestamp(action)
                if timestamp is not None:
                    live_offset = timestamp - start_time
                # compatibility with replay format
                pseudo_action = {
                    'replayChatItemAction': {'actions': [action]},
                    'videoOffsetTimeMsec': str(live_offset),
                    'isLive': True,
                }
                processed_fragment.extend(
                    json.dumps(pseudo_action, ensure_ascii=False).encode('utf-8') + b'\n')
            continuation_data_getters = [
                lambda x: x['continuations'][0]['invalidationContinuationData'],
                lambda x: x['continuations'][0]['timedContinuationData'],
            ]
            continuation_data = try_get(live_chat_continuation, continuation_data_getters, dict)
            if continuation_data:
                continuation_id = continuation_data.get('continuation')
                click_tracking_params = continuation_data.get('clickTrackingParams')
                timeout_ms = int_or_none(continuation_data.get('timeoutMs'))
                if timeout_ms is not None:
                    time.sleep(timeout_ms / 1000)
            self._append_fragment(ctx, processed_fragment)
            return continuation_id, live_offset, click_tracking_params

        def download_and_parse_fragment(url, frag_index, request_data=None, headers=None):
            count = 0
            while count <= fragment_retries:
                try:
                    success, raw_fragment = dl_fragment(url, request_data, headers)
                    if not success:
                        return False, None, None, None
                    try:
                        data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
                    except RegexNotFoundError:
                        data = None
                    if not data:
                        data = json.loads(raw_fragment)
                    live_chat_continuation = try_get(
                        data,
                        lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
                    if info_dict['protocol'] == 'youtube_live_chat_replay':
                        if frag_index == 1:
                            continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation)
                        else:
                            continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation)
                    elif info_dict['protocol'] == 'youtube_live_chat':
                        continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
                    return True, continuation_id, offset, click_tracking_params
                except compat_urllib_error.HTTPError as err:
                    count += 1
                    if count <= fragment_retries:
                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
            if count > fragment_retries:
                self.report_error('giving up after %s fragment retries' % fragment_retries)
                return False, None, None, None

        self._prepare_and_start_frag_download(ctx)

        success, raw_fragment = dl_fragment(info_dict['url'])
        if not success:
            return False
        try:
            data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
        except RegexNotFoundError:
            return False
        continuation_id = try_get(
            data,
            lambda x: x['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'])
        # no data yet but required to call _append_fragment
        self._append_fragment(ctx, b'')

        ytcfg = ie.extract_ytcfg(video_id, raw_fragment.decode('utf-8', 'replace'))

        if not ytcfg:
            return False
        api_key = try_get(ytcfg, lambda x: x['INNERTUBE_API_KEY'])
        innertube_context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'])
        if not api_key or not innertube_context:
            return False
        visitor_data = try_get(innertube_context, lambda x: x['client']['visitorData'], str)
        if info_dict['protocol'] == 'youtube_live_chat_replay':
            url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay?key=' + api_key
            chat_page_url = 'https://www.youtube.com/live_chat_replay?continuation=' + continuation_id
        elif info_dict['protocol'] == 'youtube_live_chat':
            url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat?key=' + api_key
            chat_page_url = 'https://www.youtube.com/live_chat?continuation=' + continuation_id

        frag_index = offset = 0
        click_tracking_params = None
        while continuation_id is not None:
            frag_index += 1
            request_data = {
                'context': innertube_context,
                'continuation': continuation_id,
            }
            if frag_index > 1:
                request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))}
                if click_tracking_params:
                    request_data['context']['clickTracking'] = {'clickTrackingParams': click_tracking_params}
                headers = ie.generate_api_headers(ytcfg, visitor_data=visitor_data)
                headers.update({'content-type': 'application/json'})
                fragment_request_data = json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n'
                success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
                    url, frag_index, fragment_request_data, headers)
            else:
                success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
                    chat_page_url, frag_index)
            if not success:
                return False
            if test:
                break

        self._finish_frag_download(ctx)
        return True

    @staticmethod
    def parse_live_timestamp(action):
        action_content = dict_get(
            action,
            ['addChatItemAction', 'addLiveChatTickerItemAction', 'addBannerToLiveChatCommand'])
        if not isinstance(action_content, dict):
            return None
        item = dict_get(action_content, ['item', 'bannerRenderer'])
        if not isinstance(item, dict):
            return None
        renderer = dict_get(item, [
            # text
            'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
            'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
            # ticker
            'liveChatTickerPaidMessageItemRenderer',
            'liveChatTickerSponsorItemRenderer',
            # banner
            'liveChatBannerRenderer',
        ])
        if not isinstance(renderer, dict):
            return None
        parent_item_getters = [
            lambda x: x['showItemEndpoint']['showLiveChatItemEndpoint']['renderer'],
            lambda x: x['contents'],
        ]
        parent_item = try_get(renderer, parent_item_getters, dict)
        if parent_item:
            renderer = dict_get(parent_item, [
                'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
                'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
            ])
            if not isinstance(renderer, dict):
                return None
        return int_or_none(renderer.get('timestampUsec'), 1000)
Commit	Line	Data
a78e3a57	1	from __future__ import division, unicode_literals
a78e3a57	2
a78e3a57	3	import json
c60ee3a2	4	import time
a78e3a57	5
a78e3a57	6	from .fragment import FragmentFD
82e3f6eb	7	from ..compat import compat_urllib_error
273762c8	8	from ..utils import (
273762c8	9	try_get,
c60ee3a2	10	dict_get,
c60ee3a2	11	int_or_none,
273762c8	12	RegexNotFoundError,
273762c8	13	)
82e3f6eb	14	from ..extractor.youtube import YoutubeBaseInfoExtractor as YT_BaseIE
a78e3a57	15
a78e3a57	16
c60ee3a2	17	class YoutubeLiveChatFD(FragmentFD):
c60ee3a2	18	""" Downloads YouTube live chats fragment by fragment """
a78e3a57	19
c60ee3a2	20	FD_NAME = 'youtube_live_chat'
a78e3a57	21
	22	def real_download(self, filename, info_dict):
	23	video_id = info_dict['video_id']
	24	self.to_screen('[%s] Downloading live chat' % self.FD_NAME)
	25
82e3f6eb	26	fragment_retries = self.params.get('fragment_retries', 0)
a78e3a57	27	test = self.params.get('test', False)
	28
	29	ctx = {
	30	'filename': filename,
	31	'live': True,
	32	'total_frags': None,
	33	}
	34
273762c8	35	ie = YT_BaseIE(self.ydl)
a78e3a57	36
c60ee3a2	37	start_time = int(time.time() * 1000)
c60ee3a2	38
273762c8	39	def dl_fragment(url, data=None, headers=None):
	40	http_headers = info_dict.get('http_headers', {})
	41	if headers:
	42	http_headers = http_headers.copy()
	43	http_headers.update(headers)
	44	return self._download_fragment(ctx, url, info_dict, http_headers, data)
a78e3a57	45
c60ee3a2	46	def parse_actions_replay(live_chat_continuation):
c2603313	47	offset = continuation_id = click_tracking_params = None
c60ee3a2	48	processed_fragment = bytearray()
	49	for action in live_chat_continuation.get('actions', []):
	50	if 'replayChatItemAction' in action:
	51	replay_chat_item_action = action['replayChatItemAction']
	52	offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
	53	processed_fragment.extend(
	54	json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
	55	if offset is not None:
c2603313	56	continuation = try_get(
c60ee3a2	57	live_chat_continuation,
c2603313	58	lambda x: x['continuations'][0]['liveChatReplayContinuationData'], dict)
	59	if continuation:
	60	continuation_id = continuation.get('continuation')
	61	click_tracking_params = continuation.get('clickTrackingParams')
c60ee3a2	62	self._append_fragment(ctx, processed_fragment)
c2603313	63	return continuation_id, offset, click_tracking_params
c60ee3a2	64
d534c452	65	def try_refresh_replay_beginning(live_chat_continuation):
d534c452	66	# choose the second option that contains the unfiltered live chat replay
c2603313	67	refresh_continuation = try_get(
d534c452	68	live_chat_continuation,
c2603313	69	lambda x: x['header']['liveChatHeaderRenderer']['viewSelector']['sortFilterSubMenuRenderer']['subMenuItems'][1]['continuation']['reloadContinuationData'], dict)
c2603313	70	if refresh_continuation:
d534c452	71	# no data yet but required to call _append_fragment
d534c452	72	self._append_fragment(ctx, b'')
c2603313	73	refresh_continuation_id = refresh_continuation.get('continuation')
	74	offset = 0
	75	click_tracking_params = refresh_continuation.get('trackingParams')
	76	return refresh_continuation_id, offset, click_tracking_params
d534c452	77	return parse_actions_replay(live_chat_continuation)
d534c452	78
c60ee3a2	79	live_offset = 0
	80
	81	def parse_actions_live(live_chat_continuation):
	82	nonlocal live_offset
c2603313	83	continuation_id = click_tracking_params = None
c60ee3a2	84	processed_fragment = bytearray()
	85	for action in live_chat_continuation.get('actions', []):
	86	timestamp = self.parse_live_timestamp(action)
	87	if timestamp is not None:
	88	live_offset = timestamp - start_time
	89	# compatibility with replay format
	90	pseudo_action = {
	91	'replayChatItemAction': {'actions': [action]},
	92	'videoOffsetTimeMsec': str(live_offset),
	93	'isLive': True,
	94	}
	95	processed_fragment.extend(
	96	json.dumps(pseudo_action, ensure_ascii=False).encode('utf-8') + b'\n')
	97	continuation_data_getters = [
	98	lambda x: x['continuations'][0]['invalidationContinuationData'],
	99	lambda x: x['continuations'][0]['timedContinuationData'],
	100	]
	101	continuation_data = try_get(live_chat_continuation, continuation_data_getters, dict)
	102	if continuation_data:
	103	continuation_id = continuation_data.get('continuation')
c2603313	104	click_tracking_params = continuation_data.get('clickTrackingParams')
c60ee3a2	105	timeout_ms = int_or_none(continuation_data.get('timeoutMs'))
	106	if timeout_ms is not None:
	107	time.sleep(timeout_ms / 1000)
	108	self._append_fragment(ctx, processed_fragment)
c2603313	109	return continuation_id, live_offset, click_tracking_params
c60ee3a2	110
d534c452	111	def download_and_parse_fragment(url, frag_index, request_data=None, headers=None):
82e3f6eb	112	count = 0
	113	while count <= fragment_retries:
	114	try:
c60ee3a2	115	success, raw_fragment = dl_fragment(url, request_data, headers)
82e3f6eb	116	if not success:
c2603313	117	return False, None, None, None
d534c452	118	try:
11f9be09	119	data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
d534c452	120	except RegexNotFoundError:
	121	data = None
	122	if not data:
	123	data = json.loads(raw_fragment)
82e3f6eb	124	live_chat_continuation = try_get(
	125	data,
	126	lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
d534c452	127	if info_dict['protocol'] == 'youtube_live_chat_replay':
d534c452	128	if frag_index == 1:
c2603313	129	continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation)
d534c452	130	else:
c2603313	131	continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation)
d534c452	132	elif info_dict['protocol'] == 'youtube_live_chat':
c2603313	133	continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
c2603313	134	return True, continuation_id, offset, click_tracking_params
82e3f6eb	135	except compat_urllib_error.HTTPError as err:
	136	count += 1
	137	if count <= fragment_retries:
	138	self.report_retry_fragment(err, frag_index, count, fragment_retries)
	139	if count > fragment_retries:
	140	self.report_error('giving up after %s fragment retries' % fragment_retries)
c2603313	141	return False, None, None, None
82e3f6eb	142
a78e3a57	143	self._prepare_and_start_frag_download(ctx)
a78e3a57	144
83b20a97	145	success, raw_fragment = dl_fragment(info_dict['url'])
a78e3a57	146	if not success:
a78e3a57	147	return False
273762c8	148	try:
11f9be09	149	data = ie.extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace'))
273762c8	150	except RegexNotFoundError:
273762c8	151	return False
82e3f6eb	152	continuation_id = try_get(
	153	data,
	154	lambda x: x['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'])
a78e3a57	155	# no data yet but required to call _append_fragment
	156	self._append_fragment(ctx, b'')
	157
11f9be09	158	ytcfg = ie.extract_ytcfg(video_id, raw_fragment.decode('utf-8', 'replace'))
273762c8	159
	160	if not ytcfg:
	161	return False
	162	api_key = try_get(ytcfg, lambda x: x['INNERTUBE_API_KEY'])
	163	innertube_context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'])
	164	if not api_key or not innertube_context:
	165	return False
c60ee3a2	166	visitor_data = try_get(innertube_context, lambda x: x['client']['visitorData'], str)
	167	if info_dict['protocol'] == 'youtube_live_chat_replay':
	168	url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay?key=' + api_key
d534c452	169	chat_page_url = 'https://www.youtube.com/live_chat_replay?continuation=' + continuation_id
c60ee3a2	170	elif info_dict['protocol'] == 'youtube_live_chat':
c60ee3a2	171	url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat?key=' + api_key
d534c452	172	chat_page_url = 'https://www.youtube.com/live_chat?continuation=' + continuation_id
273762c8	173
82e3f6eb	174	frag_index = offset = 0
c2603313	175	click_tracking_params = None
a78e3a57	176	while continuation_id is not None:
82e3f6eb	177	frag_index += 1
273762c8	178	request_data = {
	179	'context': innertube_context,
	180	'continuation': continuation_id,
	181	}
	182	if frag_index > 1:
	183	request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))}
c2603313	184	if click_tracking_params:
c2603313	185	request_data['context']['clickTracking'] = {'clickTrackingParams': click_tracking_params}
11f9be09	186	headers = ie.generate_api_headers(ytcfg, visitor_data=visitor_data)
d534c452	187	headers.update({'content-type': 'application/json'})
d534c452	188	fragment_request_data = json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n'
c2603313	189	success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
d534c452	190	url, frag_index, fragment_request_data, headers)
d534c452	191	else:
c2603313	192	success, continuation_id, offset, click_tracking_params = download_and_parse_fragment(
c2603313	193	chat_page_url, frag_index)
82e3f6eb	194	if not success:
	195	return False
	196	if test:
a78e3a57	197	break
	198
	199	self._finish_frag_download(ctx)
a78e3a57	200	return True
c60ee3a2	201
	202	@staticmethod
	203	def parse_live_timestamp(action):
	204	action_content = dict_get(
	205	action,
	206	['addChatItemAction', 'addLiveChatTickerItemAction', 'addBannerToLiveChatCommand'])
	207	if not isinstance(action_content, dict):
	208	return None
	209	item = dict_get(action_content, ['item', 'bannerRenderer'])
	210	if not isinstance(item, dict):
	211	return None
	212	renderer = dict_get(item, [
	213	# text
	214	'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
	215	'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
	216	# ticker
	217	'liveChatTickerPaidMessageItemRenderer',
	218	'liveChatTickerSponsorItemRenderer',
	219	# banner
	220	'liveChatBannerRenderer',
	221	])
	222	if not isinstance(renderer, dict):
	223	return None
	224	parent_item_getters = [
	225	lambda x: x['showItemEndpoint']['showLiveChatItemEndpoint']['renderer'],
	226	lambda x: x['contents'],
	227	]
	228	parent_item = try_get(renderer, parent_item_getters, dict)
	229	if parent_item:
	230	renderer = dict_get(parent_item, [
	231	'liveChatTextMessageRenderer', 'liveChatPaidMessageRenderer',
	232	'liveChatMembershipItemRenderer', 'liveChatPaidStickerRenderer',
	233	])
	234	if not isinstance(renderer, dict):
	235	return None
	236	return int_or_none(renderer.get('timestampUsec'), 1000)