]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/livestream.py
[extractor] Deprecate `_sort_formats`
[yt-dlp.git] / yt_dlp / extractor / livestream.py
CommitLineData
b4444d5c 1import re
082b1155 2import itertools
b4444d5c
JMF
3
4from .common import InfoExtractor
1cc79574 5from ..compat import (
cbf915f3 6 compat_str,
b00ca882 7 compat_urlparse,
1cc79574
PH
8)
9from ..utils import (
cbf915f3 10 find_xpath_attr,
64ccbf18 11 xpath_attr,
cbf915f3 12 xpath_with_ns,
64ccbf18 13 xpath_text,
14 orderedSet,
e375a149 15 update_url_query,
64ccbf18 16 int_or_none,
17 float_or_none,
18 parse_iso8601,
19 determine_ext,
b00ca882 20)
b4444d5c
JMF
21
22
23class LivestreamIE(InfoExtractor):
c5469e04 24 IE_NAME = 'livestream'
64ccbf18 25 _VALID_URL = r'https?://(?:new\.)?livestream\.com/(?:accounts/(?P<account_id>\d+)|(?P<account_name>[^/]+))/(?:events/(?P<event_id>\d+)|(?P<event_name>[^/]+))(?:/videos/(?P<id>\d+))?'
bfd973ec 26 _EMBED_REGEX = [r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"']
27
22a6f150 28 _TESTS = [{
c5469e04
S
29 'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
30 'md5': '53274c76ba7754fb0e8d072716f2292b',
31 'info_dict': {
32 'id': '4719370',
33 'ext': 'mp4',
34 'title': 'Live from Webster Hall NYC',
64ccbf18 35 'timestamp': 1350008072,
c5469e04 36 'upload_date': '20121012',
64ccbf18 37 'duration': 5968.0,
cbf915f3
PH
38 'like_count': int,
39 'view_count': int,
ec85ded8 40 'thumbnail': r're:^http://.*\.jpg$'
b4444d5c 41 }
22a6f150
PH
42 }, {
43 'url': 'http://new.livestream.com/tedx/cityenglish',
44 'info_dict': {
45 'title': 'TEDCity2.0 (English)',
1def5f35 46 'id': '2245590',
22a6f150
PH
47 },
48 'playlist_mincount': 4,
082b1155
JMF
49 }, {
50 'url': 'http://new.livestream.com/chess24/tatasteelchess',
51 'info_dict': {
52 'title': 'Tata Steel Chess',
53 'id': '3705884',
54 },
55 'playlist_mincount': 60,
af63fed7
PH
56 }, {
57 'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
58 'only_matching': True,
4a20c9f6
YCH
59 }, {
60 'url': 'http://livestream.com/bsww/concacafbeachsoccercampeonato2015',
61 'only_matching': True,
22a6f150 62 }]
64ccbf18 63 _API_URL_TEMPLATE = 'http://livestream.com/api/accounts/%s/events/%s'
64
65 def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
5b025168 66 base_ele = find_xpath_attr(
67 smil, self._xpath_ns('.//meta', namespace), 'name', 'httpBase')
e1dd521e 68 base = base_ele.get('content') if base_ele is not None else 'http://livestreamvod-f.akamaihd.net/'
b4444d5c 69
8f3034d8 70 formats = []
64ccbf18 71 video_nodes = smil.findall(self._xpath_ns('.//video', namespace))
8f3034d8
PH
72
73 for vn in video_nodes:
64ccbf18 74 tbr = int_or_none(vn.attrib.get('system-bitrate'), 1000)
8f3034d8 75 furl = (
e375a149
S
76 update_url_query(compat_urlparse.urljoin(base, vn.attrib['src']), {
77 'v': '3.0.3',
78 'fp': 'WIN% 14,0,0,145',
79 }))
8f3034d8
PH
80 if 'clipBegin' in vn.attrib:
81 furl += '&ssek=' + vn.attrib['clipBegin']
82 formats.append({
83 'url': furl,
84 'format_id': 'smil_%d' % tbr,
85 'ext': 'flv',
86 'tbr': tbr,
f983b875 87 'preference': -1000, # Strictly inferior than all other formats?
8f3034d8
PH
88 })
89 return formats
90
b4444d5c 91 def _extract_video_info(self, video_data):
cbf915f3
PH
92 video_id = compat_str(video_data['id'])
93
94 FORMAT_KEYS = (
95 ('sd', 'progressive_url'),
96 ('hd', 'progressive_url_hd'),
72e785f3 97 )
64ccbf18 98
99 formats = []
100 for format_id, key in FORMAT_KEYS:
101 video_url = video_data.get(key)
102 if video_url:
103 ext = determine_ext(video_url)
7b813165 104 if ext == 'm3u8':
105 continue
5b025168 106 bitrate = int_or_none(self._search_regex(
107 r'(\d+)\.%s' % ext, video_url, 'bitrate', default=None))
64ccbf18 108 formats.append({
109 'url': video_url,
110 'format_id': format_id,
111 'tbr': bitrate,
112 'ext': ext,
113 })
cbf915f3
PH
114
115 smil_url = video_data.get('smil_url')
116 if smil_url:
a9efdf3d 117 formats.extend(self._extract_smil_formats(smil_url, video_id, fatal=False))
64ccbf18 118
119 m3u8_url = video_data.get('m3u8_url')
120 if m3u8_url:
7e5edcfd 121 formats.extend(self._extract_m3u8_formats(
fb4fc449
RA
122 m3u8_url, video_id, 'mp4', 'm3u8_native',
123 m3u8_id='hls', fatal=False))
64ccbf18 124
125 f4m_url = video_data.get('f4m_url')
126 if f4m_url:
7e5edcfd
S
127 formats.extend(self._extract_f4m_formats(
128 f4m_url, video_id, f4m_id='hds', fatal=False))
cbf915f3 129
64ccbf18 130 comments = [{
131 'author_id': comment.get('author_id'),
132 'author': comment.get('author', {}).get('full_name'),
133 'id': comment.get('id'),
134 'text': comment['text'],
135 'timestamp': parse_iso8601(comment.get('created_at')),
136 } for comment in video_data.get('comments', {}).get('data', [])]
137
c5469e04 138 return {
cbf915f3
PH
139 'id': video_id,
140 'formats': formats,
c5469e04 141 'title': video_data['caption'],
64ccbf18 142 'description': video_data.get('description'),
cbf915f3 143 'thumbnail': video_data.get('thumbnail_url'),
64ccbf18 144 'duration': float_or_none(video_data.get('duration'), 1000),
145 'timestamp': parse_iso8601(video_data.get('publish_at')),
cbf915f3 146 'like_count': video_data.get('likes', {}).get('total'),
64ccbf18 147 'comment_count': video_data.get('comments', {}).get('total'),
cbf915f3 148 'view_count': video_data.get('views'),
64ccbf18 149 'comments': comments,
150 }
151
152 def _extract_stream_info(self, stream_info):
7a46542f 153 broadcast_id = compat_str(stream_info['broadcast_id'])
64ccbf18 154 is_live = stream_info.get('is_live')
155
156 formats = []
157 smil_url = stream_info.get('play_url')
158 if smil_url:
7e5edcfd 159 formats.extend(self._extract_smil_formats(smil_url, broadcast_id))
64ccbf18 160
161 m3u8_url = stream_info.get('m3u8_url')
162 if m3u8_url:
7e5edcfd 163 formats.extend(self._extract_m3u8_formats(
fb4fc449
RA
164 m3u8_url, broadcast_id, 'mp4', 'm3u8_native',
165 m3u8_id='hls', fatal=False))
64ccbf18 166
167 rtsp_url = stream_info.get('rtsp_url')
168 if rtsp_url:
169 formats.append({
170 'url': rtsp_url,
171 'format_id': 'rtsp',
172 })
64ccbf18 173
174 return {
175 'id': broadcast_id,
176 'formats': formats,
39ca3b5c 177 'title': stream_info['stream_title'],
64ccbf18 178 'thumbnail': stream_info.get('thumbnail_url'),
179 'is_live': is_live,
c5469e04 180 }
b4444d5c 181
64ccbf18 182 def _extract_event(self, event_data):
183 event_id = compat_str(event_data['id'])
184 account_id = compat_str(event_data['owner_account_id'])
185 feed_root_url = self._API_URL_TEMPLATE % (account_id, event_id) + '/feed.json'
186
187 stream_info = event_data.get('stream_info')
188 if stream_info:
189 return self._extract_stream_info(stream_info)
190
191 last_video = None
192 entries = []
193 for i in itertools.count(1):
194 if last_video is None:
195 info_url = feed_root_url
196 else:
197 info_url = '{root}?&id={id}&newer=-1&type=video'.format(
198 root=feed_root_url, id=last_video)
5b025168 199 videos_info = self._download_json(
200 info_url, event_id, 'Downloading page {0}'.format(i))['data']
64ccbf18 201 videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
202 if not videos_info:
203 break
204 for v in videos_info:
a26a9d62 205 v_id = compat_str(v['id'])
64ccbf18 206 entries.append(self.url_result(
a26a9d62
S
207 'http://livestream.com/accounts/%s/events/%s/videos/%s' % (account_id, event_id, v_id),
208 'Livestream', v_id, v.get('caption')))
64ccbf18 209 last_video = videos_info[-1]['id']
210 return self.playlist_result(entries, event_id, event_data['full_name'])
082b1155 211
b4444d5c 212 def _real_extract(self, url):
5ad28e7f 213 mobj = self._match_valid_url(url)
b4444d5c 214 video_id = mobj.group('id')
64ccbf18 215 event = mobj.group('event_id') or mobj.group('event_name')
216 account = mobj.group('account_id') or mobj.group('account_name')
217 api_url = self._API_URL_TEMPLATE % (account, event)
218 if video_id:
5b025168 219 video_data = self._download_json(
220 api_url + '/videos/%s' % video_id, video_id)
64ccbf18 221 return self._extract_video_info(video_data)
22a6f150 222 else:
64ccbf18 223 event_data = self._download_json(api_url, video_id)
224 return self._extract_event(event_data)
c66d2baa
JMF
225
226
227# The original version of Livestream uses a different system
228class LivestreamOriginalIE(InfoExtractor):
c5469e04 229 IE_NAME = 'livestream:original'
a9055266 230 _VALID_URL = r'''(?x)https?://original\.livestream\.com/
5b025168 231 (?P<user>[^/\?#]+)(?:/(?P<type>video|folder)
232 (?:(?:\?.*?Id=|/)(?P<id>.*?)(&|$))?)?
78338f71 233 '''
22a6f150 234 _TESTS = [{
a9055266 235 'url': 'http://original.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
c5469e04
S
236 'info_dict': {
237 'id': 'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
883340c1 238 'ext': 'mp4',
c5469e04 239 'title': 'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
64ccbf18 240 'duration': 771.301,
241 'view_count': int,
c66d2baa 242 },
22a6f150 243 }, {
a9055266 244 'url': 'https://original.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3',
22a6f150
PH
245 'info_dict': {
246 'id': 'a07bf706-d0e4-4e75-a747-b021d84f2fd3',
247 },
248 'playlist_mincount': 4,
5b025168 249 }, {
250 # live stream
c71d2e20 251 'url': 'http://original.livestream.com/znsbahamas',
5b025168 252 'only_matching': True,
22a6f150 253 }]
c66d2baa 254
5b025168 255 def _extract_video_info(self, user, video_id):
256 api_url = 'http://x%sx.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id=%s' % (user, video_id)
e26f8712 257 info = self._download_xml(api_url, video_id)
5b025168 258
c66d2baa 259 item = info.find('channel').find('item')
5b025168 260 title = xpath_text(item, 'title')
64ccbf18 261 media_ns = {'media': 'http://search.yahoo.com/mrss'}
5b025168 262 thumbnail_url = xpath_attr(
263 item, xpath_with_ns('media:thumbnail', media_ns), 'url')
264 duration = float_or_none(xpath_attr(
265 item, xpath_with_ns('media:content', media_ns), 'duration'))
64ccbf18 266 ls_ns = {'ls': 'http://api.channel.livestream.com/2.0'}
5b025168 267 view_count = int_or_none(xpath_text(
268 item, xpath_with_ns('ls:viewsCount', ls_ns)))
64ccbf18 269
5b025168 270 return {
271 'id': video_id,
272 'title': title,
273 'thumbnail': thumbnail_url,
274 'duration': duration,
275 'view_count': view_count,
276 }
277
fb4fc449 278 def _extract_video_formats(self, video_data, video_id):
5b025168 279 formats = []
280
281 progressive_url = video_data.get('progressiveUrl')
282 if progressive_url:
283 formats.append({
284 'url': progressive_url,
285 'format_id': 'http',
286 })
64ccbf18 287
5b025168 288 m3u8_url = video_data.get('httpUrl')
64ccbf18 289 if m3u8_url:
7e5edcfd 290 formats.extend(self._extract_m3u8_formats(
fb4fc449
RA
291 m3u8_url, video_id, 'mp4', 'm3u8_native',
292 m3u8_id='hls', fatal=False))
64ccbf18 293
5b025168 294 rtsp_url = video_data.get('rtspUrl')
64ccbf18 295 if rtsp_url:
296 formats.append({
297 'url': rtsp_url,
298 'format_id': 'rtsp',
299 })
c66d2baa 300
5b025168 301 return formats
78338f71
JMF
302
303 def _extract_folder(self, url, folder_id):
304 webpage = self._download_webpage(url, folder_id)
22a6f150
PH
305 paths = orderedSet(re.findall(
306 r'''(?x)(?:
307 <li\s+class="folder">\s*<a\s+href="|
308 <a\s+href="(?=https?://livestre\.am/)
309 )([^"]+)"''', webpage))
78338f71 310
64ccbf18 311 entries = [{
312 '_type': 'url',
313 'url': compat_urlparse.urljoin(url, p),
314 } for p in paths]
315
316 return self.playlist_result(entries, folder_id)
78338f71
JMF
317
318 def _real_extract(self, url):
5ad28e7f 319 mobj = self._match_valid_url(url)
78338f71
JMF
320 user = mobj.group('user')
321 url_type = mobj.group('type')
5b025168 322 content_id = mobj.group('id')
78338f71 323 if url_type == 'folder':
5b025168 324 return self._extract_folder(url, content_id)
78338f71 325 else:
5b025168 326 # this url is used on mobile devices
327 stream_url = 'http://x%sx.api.channel.livestream.com/3.0/getstream.json' % user
328 info = {}
329 if content_id:
330 stream_url += '?id=%s' % content_id
331 info = self._extract_video_info(user, content_id)
332 else:
333 content_id = user
334 webpage = self._download_webpage(url, content_id)
335 info = {
336 'title': self._og_search_title(webpage),
337 'description': self._og_search_description(webpage),
197224b7 338 'thumbnail': self._search_regex(r'channelLogo\.src\s*=\s*"([^"]+)"', webpage, 'thumbnail', None),
5b025168 339 }
340 video_data = self._download_json(stream_url, content_id)
341 is_live = video_data.get('isLive')
5b025168 342 info.update({
343 'id': content_id,
39ca3b5c 344 'title': info['title'],
fb4fc449 345 'formats': self._extract_video_formats(video_data, content_id),
5b025168 346 'is_live': is_live,
347 })
348 return info
78338f71
JMF
349
350
351# The server doesn't support HEAD request, the generic extractor can't detect
352# the redirection
353class LivestreamShortenerIE(InfoExtractor):
354 IE_NAME = 'livestream:shortener'
355 IE_DESC = False # Do not list
356 _VALID_URL = r'https?://livestre\.am/(?P<id>.+)'
357
358 def _real_extract(self, url):
5ad28e7f 359 mobj = self._match_valid_url(url)
78338f71
JMF
360 id = mobj.group('id')
361 webpage = self._download_webpage(url, id)
362
d226c560 363 return self.url_result(self._og_search_url(webpage))