]>
Commit | Line | Data |
---|---|---|
a78e3a57 | 1 | from __future__ import division, unicode_literals |
2 | ||
a78e3a57 | 3 | import json |
4 | ||
5 | from .fragment import FragmentFD | |
82e3f6eb | 6 | from ..compat import compat_urllib_error |
273762c8 | 7 | from ..utils import ( |
8 | try_get, | |
9 | RegexNotFoundError, | |
10 | ) | |
82e3f6eb | 11 | from ..extractor.youtube import YoutubeBaseInfoExtractor as YT_BaseIE |
a78e3a57 | 12 | |
13 | ||
14 | class YoutubeLiveChatReplayFD(FragmentFD): | |
15 | """ Downloads YouTube live chat replays fragment by fragment """ | |
16 | ||
17 | FD_NAME = 'youtube_live_chat_replay' | |
18 | ||
19 | def real_download(self, filename, info_dict): | |
20 | video_id = info_dict['video_id'] | |
21 | self.to_screen('[%s] Downloading live chat' % self.FD_NAME) | |
22 | ||
82e3f6eb | 23 | fragment_retries = self.params.get('fragment_retries', 0) |
a78e3a57 | 24 | test = self.params.get('test', False) |
25 | ||
26 | ctx = { | |
27 | 'filename': filename, | |
28 | 'live': True, | |
29 | 'total_frags': None, | |
30 | } | |
31 | ||
273762c8 | 32 | ie = YT_BaseIE(self.ydl) |
a78e3a57 | 33 | |
273762c8 | 34 | def dl_fragment(url, data=None, headers=None): |
35 | http_headers = info_dict.get('http_headers', {}) | |
36 | if headers: | |
37 | http_headers = http_headers.copy() | |
38 | http_headers.update(headers) | |
39 | return self._download_fragment(ctx, url, info_dict, http_headers, data) | |
a78e3a57 | 40 | |
273762c8 | 41 | def download_and_parse_fragment(url, frag_index, request_data): |
82e3f6eb | 42 | count = 0 |
43 | while count <= fragment_retries: | |
44 | try: | |
273762c8 | 45 | success, raw_fragment = dl_fragment(url, request_data, {'content-type': 'application/json'}) |
82e3f6eb | 46 | if not success: |
47 | return False, None, None | |
273762c8 | 48 | try: |
49 | data = ie._extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace')) | |
50 | except RegexNotFoundError: | |
51 | data = None | |
4d608b52 | 52 | if not data: |
273762c8 | 53 | data = json.loads(raw_fragment) |
82e3f6eb | 54 | live_chat_continuation = try_get( |
55 | data, | |
56 | lambda x: x['continuationContents']['liveChatContinuation'], dict) or {} | |
57 | offset = continuation_id = None | |
58 | processed_fragment = bytearray() | |
59 | for action in live_chat_continuation.get('actions', []): | |
60 | if 'replayChatItemAction' in action: | |
61 | replay_chat_item_action = action['replayChatItemAction'] | |
62 | offset = int(replay_chat_item_action['videoOffsetTimeMsec']) | |
63 | processed_fragment.extend( | |
64 | json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n') | |
65 | if offset is not None: | |
66 | continuation_id = try_get( | |
67 | live_chat_continuation, | |
68 | lambda x: x['continuations'][0]['liveChatReplayContinuationData']['continuation']) | |
69 | self._append_fragment(ctx, processed_fragment) | |
70 | ||
71 | return True, continuation_id, offset | |
72 | except compat_urllib_error.HTTPError as err: | |
73 | count += 1 | |
74 | if count <= fragment_retries: | |
75 | self.report_retry_fragment(err, frag_index, count, fragment_retries) | |
76 | if count > fragment_retries: | |
77 | self.report_error('giving up after %s fragment retries' % fragment_retries) | |
78 | return False, None, None | |
79 | ||
a78e3a57 | 80 | self._prepare_and_start_frag_download(ctx) |
81 | ||
82 | success, raw_fragment = dl_fragment( | |
83 | 'https://www.youtube.com/watch?v={}'.format(video_id)) | |
84 | if not success: | |
85 | return False | |
273762c8 | 86 | try: |
87 | data = ie._extract_yt_initial_data(video_id, raw_fragment.decode('utf-8', 'replace')) | |
88 | except RegexNotFoundError: | |
89 | return False | |
82e3f6eb | 90 | continuation_id = try_get( |
91 | data, | |
92 | lambda x: x['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']) | |
a78e3a57 | 93 | # no data yet but required to call _append_fragment |
94 | self._append_fragment(ctx, b'') | |
95 | ||
273762c8 | 96 | ytcfg = ie._extract_ytcfg(video_id, raw_fragment.decode('utf-8', 'replace')) |
97 | ||
98 | if not ytcfg: | |
99 | return False | |
100 | api_key = try_get(ytcfg, lambda x: x['INNERTUBE_API_KEY']) | |
101 | innertube_context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT']) | |
102 | if not api_key or not innertube_context: | |
103 | return False | |
104 | url = 'https://www.youtube.com/youtubei/v1/live_chat/get_live_chat_replay?key=' + api_key | |
105 | ||
82e3f6eb | 106 | frag_index = offset = 0 |
a78e3a57 | 107 | while continuation_id is not None: |
82e3f6eb | 108 | frag_index += 1 |
273762c8 | 109 | request_data = { |
110 | 'context': innertube_context, | |
111 | 'continuation': continuation_id, | |
112 | } | |
113 | if frag_index > 1: | |
114 | request_data['currentPlayerState'] = {'playerOffsetMs': str(max(offset - 5000, 0))} | |
115 | success, continuation_id, offset = download_and_parse_fragment( | |
116 | url, frag_index, json.dumps(request_data, ensure_ascii=False).encode('utf-8') + b'\n') | |
82e3f6eb | 117 | if not success: |
118 | return False | |
119 | if test: | |
a78e3a57 | 120 | break |
121 | ||
122 | self._finish_frag_download(ctx) | |
a78e3a57 | 123 | return True |