youtube_dlc/downloader/youtube_live_chat.py

   1 from __future__ import division, unicode_literals
   2
   3 import re
   4 import json
   5
   6 from .fragment import FragmentFD
   7 from ..compat import compat_urllib_error
   8 from ..utils import try_get
   9 from ..extractor.youtube import YoutubeBaseInfoExtractor as YT_BaseIE
  10
  11
  12 class YoutubeLiveChatReplayFD(FragmentFD):
  13     """ Downloads YouTube live chat replays fragment by fragment """
  14
  15     FD_NAME = 'youtube_live_chat_replay'
  16
  17     def real_download(self, filename, info_dict):
  18         video_id = info_dict['video_id']
  19         self.to_screen('[%s] Downloading live chat' % self.FD_NAME)
  20
  21         fragment_retries = self.params.get('fragment_retries', 0)
  22         test = self.params.get('test', False)
  23
  24         ctx = {
  25             'filename': filename,
  26             'live': True,
  27             'total_frags': None,
  28         }
  29
  30         def dl_fragment(url):
  31             headers = info_dict.get('http_headers', {})
  32             return self._download_fragment(ctx, url, info_dict, headers)
  33
  34         def parse_yt_initial_data(data):
  35             patterns = (
  36                 r'%s\\s*%s' % (YT_BaseIE._YT_INITIAL_DATA_RE, YT_BaseIE._YT_INITIAL_BOUNDARY_RE),
  37                 r'%s' % YT_BaseIE._YT_INITIAL_DATA_RE)
  38             data = data.decode('utf-8', 'replace')
  39             for patt in patterns:
  40                 try:
  41                     raw_json = re.search(patt, data).group(1)
  42                     return json.loads(raw_json)
  43                 except AttributeError:
  44                     continue
  45
  46         def download_and_parse_fragment(url, frag_index):
  47             count = 0
  48             while count <= fragment_retries:
  49                 try:
  50                     success, raw_fragment = dl_fragment(url)
  51                     if not success:
  52                         return False, None, None
  53                     data = parse_yt_initial_data(raw_fragment) or json.loads(raw_fragment)['response']
  54
  55                     live_chat_continuation = try_get(
  56                         data,
  57                         lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
  58                     offset = continuation_id = None
  59                     processed_fragment = bytearray()
  60                     for action in live_chat_continuation.get('actions', []):
  61                         if 'replayChatItemAction' in action:
  62                             replay_chat_item_action = action['replayChatItemAction']
  63                             offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
  64                         processed_fragment.extend(
  65                             json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
  66                     if offset is not None:
  67                         continuation_id = try_get(
  68                             live_chat_continuation,
  69                             lambda x: x['continuations'][0]['liveChatReplayContinuationData']['continuation'])
  70                     self._append_fragment(ctx, processed_fragment)
  71
  72                     return True, continuation_id, offset
  73                 except compat_urllib_error.HTTPError as err:
  74                     count += 1
  75                     if count <= fragment_retries:
  76                         self.report_retry_fragment(err, frag_index, count, fragment_retries)
  77             if count > fragment_retries:
  78                 self.report_error('giving up after %s fragment retries' % fragment_retries)
  79                 return False, None, None
  80
  81         self._prepare_and_start_frag_download(ctx)
  82
  83         success, raw_fragment = dl_fragment(
  84             'https://www.youtube.com/watch?v={}'.format(video_id))
  85         if not success:
  86             return False
  87         data = parse_yt_initial_data(raw_fragment)
  88         continuation_id = try_get(
  89             data,
  90             lambda x: x['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'])
  91         # no data yet but required to call _append_fragment
  92         self._append_fragment(ctx, b'')
  93
  94         frag_index = offset = 0
  95         while continuation_id is not None:
  96             frag_index += 1
  97             url = ''.join((
  98                 'https://www.youtube.com/live_chat_replay',
  99                 '/get_live_chat_replay' if frag_index > 1 else '',
 100                 '?continuation=%s' % continuation_id,
 101                 '&playerOffsetMs=%d&hidden=false&pbj=1' % max(offset - 5000, 0) if frag_index > 1 else ''))
 102             success, continuation_id, offset = download_and_parse_fragment(url, frag_index)
 103             if not success:
 104                 return False
 105             if test:
 106                 break
 107
 108         self._finish_frag_download(ctx)
 109         return True