youtube_dl/downloader/youtube_live_chat.py

   1 from __future__ import division, unicode_literals
   2
   3 import re
   4 import json
   5
   6 from .fragment import FragmentFD
   7
   8
   9 class YoutubeLiveChatReplayFD(FragmentFD):
  10     """ Downloads YouTube live chat replays fragment by fragment """
  11
  12     FD_NAME = 'youtube_live_chat_replay'
  13
  14     def real_download(self, filename, info_dict):
  15         video_id = info_dict['video_id']
  16         self.to_screen('[%s] Downloading live chat' % self.FD_NAME)
  17
  18         test = self.params.get('test', False)
  19
  20         ctx = {
  21             'filename': filename,
  22             'live': True,
  23             'total_frags': None,
  24         }
  25
  26         def dl_fragment(url):
  27             headers = info_dict.get('http_headers', {})
  28             return self._download_fragment(ctx, url, info_dict, headers)
  29
  30         def parse_yt_initial_data(data):
  31             window_patt = b'window\\["ytInitialData"\\]\\s*=\\s*(.*?)(?<=});'
  32             var_patt = b'var\\s+ytInitialData\\s*=\\s*(.*?)(?<=});'
  33             for patt in window_patt, var_patt:
  34                 try:
  35                     raw_json = re.search(patt, data).group(1)
  36                     return json.loads(raw_json)
  37                 except AttributeError:
  38                     continue
  39
  40         self._prepare_and_start_frag_download(ctx)
  41
  42         success, raw_fragment = dl_fragment(
  43             'https://www.youtube.com/watch?v={}'.format(video_id))
  44         if not success:
  45             return False
  46         data = parse_yt_initial_data(raw_fragment)
  47         continuation_id = data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
  48         # no data yet but required to call _append_fragment
  49         self._append_fragment(ctx, b'')
  50
  51         first = True
  52         offset = None
  53         while continuation_id is not None:
  54             data = None
  55             if first:
  56                 url = 'https://www.youtube.com/live_chat_replay?continuation={}'.format(continuation_id)
  57                 success, raw_fragment = dl_fragment(url)
  58                 if not success:
  59                     return False
  60                 data = parse_yt_initial_data(raw_fragment)
  61             else:
  62                 url = ('https://www.youtube.com/live_chat_replay/get_live_chat_replay'
  63                        + '?continuation={}'.format(continuation_id)
  64                        + '&playerOffsetMs={}'.format(offset - 5000)
  65                        + '&hidden=false'
  66                        + '&pbj=1')
  67                 success, raw_fragment = dl_fragment(url)
  68                 if not success:
  69                     return False
  70                 data = json.loads(raw_fragment)['response']
  71
  72             first = False
  73             continuation_id = None
  74
  75             live_chat_continuation = data['continuationContents']['liveChatContinuation']
  76             offset = None
  77             processed_fragment = bytearray()
  78             if 'actions' in live_chat_continuation:
  79                 for action in live_chat_continuation['actions']:
  80                     if 'replayChatItemAction' in action:
  81                         replay_chat_item_action = action['replayChatItemAction']
  82                         offset = int(replay_chat_item_action['videoOffsetTimeMsec'])
  83                     processed_fragment.extend(
  84                         json.dumps(action, ensure_ascii=False).encode('utf-8') + b'\n')
  85                 continuation_id = live_chat_continuation['continuations'][0]['liveChatReplayContinuationData']['continuation']
  86
  87             self._append_fragment(ctx, processed_fragment)
  88
  89             if test or offset is None:
  90                 break
  91
  92         self._finish_frag_download(ctx)
  93
  94         return True