yt_dlp/extractor/periscope.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     int_or_none,
   4     parse_iso8601,
   5     unescapeHTML,
   6 )
   7 from ..utils.traversal import traverse_obj
   8
   9
  10 class PeriscopeBaseIE(InfoExtractor):
  11     _M3U8_HEADERS = {
  12         'Referer': 'https://www.periscope.tv/'
  13     }
  14
  15     def _call_api(self, method, query, item_id):
  16         return self._download_json(
  17             'https://api.periscope.tv/api/v2/%s' % method,
  18             item_id, query=query)
  19
  20     def _parse_broadcast_data(self, broadcast, video_id):
  21         title = broadcast.get('status') or 'Periscope Broadcast'
  22         uploader = broadcast.get('user_display_name') or broadcast.get('username')
  23         title = '%s - %s' % (uploader, title) if uploader else title
  24         thumbnails = [{
  25             'url': broadcast[image],
  26         } for image in ('image_url', 'image_url_medium', 'image_url_small') if broadcast.get(image)]
  27
  28         return {
  29             'id': broadcast.get('id') or video_id,
  30             'title': title,
  31             'timestamp': parse_iso8601(broadcast.get('created_at')) or int_or_none(
  32                 broadcast.get('created_at_ms'), scale=1000),
  33             'release_timestamp': int_or_none(broadcast.get('scheduled_start_ms'), scale=1000),
  34             'uploader': uploader,
  35             'uploader_id': broadcast.get('user_id') or broadcast.get('username'),
  36             'thumbnails': thumbnails,
  37             'view_count': int_or_none(broadcast.get('total_watched')),
  38             'concurrent_view_count': int_or_none(broadcast.get('total_watching')),
  39             'tags': broadcast.get('tags'),
  40             'live_status': {
  41                 'running': 'is_live',
  42                 'not_started': 'is_upcoming',
  43             }.get(traverse_obj(broadcast, ('state', {str.lower}))) or 'was_live'
  44         }
  45
  46     @staticmethod
  47     def _extract_common_format_info(broadcast):
  48         return broadcast.get('state').lower(), int_or_none(broadcast.get('width')), int_or_none(broadcast.get('height'))
  49
  50     @staticmethod
  51     def _add_width_and_height(f, width, height):
  52         for key, val in (('width', width), ('height', height)):
  53             if not f.get(key):
  54                 f[key] = val
  55
  56     def _extract_pscp_m3u8_formats(self, m3u8_url, video_id, format_id, state, width, height, fatal=True):
  57         m3u8_formats = self._extract_m3u8_formats(
  58             m3u8_url, video_id, 'mp4',
  59             entry_protocol='m3u8_native'
  60             if state in ('ended', 'timed_out') else 'm3u8',
  61             m3u8_id=format_id, fatal=fatal, headers=self._M3U8_HEADERS)
  62         if len(m3u8_formats) == 1:
  63             self._add_width_and_height(m3u8_formats[0], width, height)
  64         for f in m3u8_formats:
  65             f.setdefault('http_headers', {}).update(self._M3U8_HEADERS)
  66         return m3u8_formats
  67
  68
  69 class PeriscopeIE(PeriscopeBaseIE):
  70     IE_DESC = 'Periscope'
  71     IE_NAME = 'periscope'
  72     _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/[^/]+/(?P<id>[^/?#]+)'
  73     _EMBED_REGEX = [r'<iframe[^>]+src=([\'"])(?P<url>(?:https?:)?//(?:www\.)?(?:periscope|pscp)\.tv/(?:(?!\1).)+)\1']
  74     # Alive example URLs can be found here https://www.periscope.tv/
  75     _TESTS = [{
  76         'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
  77         'md5': '65b57957972e503fcbbaeed8f4fa04ca',
  78         'info_dict': {
  79             'id': '56102209',
  80             'ext': 'mp4',
  81             'title': 'Bec Boop - 🚠✈️🇬🇧 Fly above #London in Emirates Air Line cable car at night 🇬🇧✈️🚠 #BoopScope 🎀💗',
  82             'timestamp': 1438978559,
  83             'upload_date': '20150807',
  84             'uploader': 'Bec Boop',
  85             'uploader_id': '1465763',
  86         },
  87         'skip': 'Expires in 24 hours',
  88     }, {
  89         'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
  90         'only_matching': True,
  91     }, {
  92         'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
  93         'only_matching': True,
  94     }, {
  95         'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
  96         'only_matching': True,
  97     }]
  98
  99     def _real_extract(self, url):
 100         token = self._match_id(url)
 101
 102         stream = self._call_api(
 103             'accessVideoPublic', {'broadcast_id': token}, token)
 104
 105         broadcast = stream['broadcast']
 106         info = self._parse_broadcast_data(broadcast, token)
 107
 108         state = broadcast.get('state').lower()
 109         width = int_or_none(broadcast.get('width'))
 110         height = int_or_none(broadcast.get('height'))
 111
 112         def add_width_and_height(f):
 113             for key, val in (('width', width), ('height', height)):
 114                 if not f.get(key):
 115                     f[key] = val
 116
 117         video_urls = set()
 118         formats = []
 119         for format_id in ('replay', 'rtmp', 'hls', 'https_hls', 'lhls', 'lhlsweb'):
 120             video_url = stream.get(format_id + '_url')
 121             if not video_url or video_url in video_urls:
 122                 continue
 123             video_urls.add(video_url)
 124             if format_id != 'rtmp':
 125                 m3u8_formats = self._extract_pscp_m3u8_formats(
 126                     video_url, token, format_id, state, width, height, False)
 127                 formats.extend(m3u8_formats)
 128                 continue
 129             rtmp_format = {
 130                 'url': video_url,
 131                 'ext': 'flv' if format_id == 'rtmp' else 'mp4',
 132             }
 133             self._add_width_and_height(rtmp_format)
 134             formats.append(rtmp_format)
 135
 136         info['formats'] = formats
 137         return info
 138
 139
 140 class PeriscopeUserIE(PeriscopeBaseIE):
 141     _VALID_URL = r'https?://(?:www\.)?(?:periscope|pscp)\.tv/(?P<id>[^/]+)/?$'
 142     IE_DESC = 'Periscope user videos'
 143     IE_NAME = 'periscope:user'
 144
 145     _TEST = {
 146         'url': 'https://www.periscope.tv/LularoeHusbandMike/',
 147         'info_dict': {
 148             'id': 'LularoeHusbandMike',
 149             'title': 'LULAROE HUSBAND MIKE',
 150             'description': 'md5:6cf4ec8047768098da58e446e82c82f0',
 151         },
 152         # Periscope only shows videos in the last 24 hours, so it's possible to
 153         # get 0 videos
 154         'playlist_mincount': 0,
 155     }
 156
 157     def _real_extract(self, url):
 158         user_name = self._match_id(url)
 159
 160         webpage = self._download_webpage(url, user_name)
 161
 162         data_store = self._parse_json(
 163             unescapeHTML(self._search_regex(
 164                 r'data-store=(["\'])(?P<data>.+?)\1',
 165                 webpage, 'data store', default='{}', group='data')),
 166             user_name)
 167
 168         user = list(data_store['UserCache']['users'].values())[0]['user']
 169         user_id = user['id']
 170         session_id = data_store['SessionToken']['public']['broadcastHistory']['token']['session_id']
 171
 172         broadcasts = self._call_api(
 173             'getUserBroadcastsPublic',
 174             {'user_id': user_id, 'session_id': session_id},
 175             user_name)['broadcasts']
 176
 177         broadcast_ids = [
 178             broadcast['id'] for broadcast in broadcasts if broadcast.get('id')]
 179
 180         title = user.get('display_name') or user.get('username') or user_name
 181         description = user.get('description')
 182
 183         entries = [
 184             self.url_result(
 185                 'https://www.periscope.tv/%s/%s' % (user_name, broadcast_id))
 186             for broadcast_id in broadcast_ids]
 187
 188         return self.playlist_result(entries, user_id, title, description)