2 from __future__
import unicode_literals
8 from .common
import InfoExtractor
12 compat_urllib_parse_urlencode
,
13 compat_urllib_parse_urlparse
,
27 class TwitchBaseIE(InfoExtractor
):
28 _VALID_URL_BASE
= r
'https?://(?:www\.)?twitch\.tv'
30 _API_BASE
= 'https://api.twitch.tv'
31 _USHER_BASE
= 'http://usher.twitch.tv'
32 _LOGIN_URL
= 'http://www.twitch.tv/login'
33 _NETRC_MACHINE
= 'twitch'
35 def _handle_error(self
, response
):
36 if not isinstance(response
, dict):
38 error
= response
.get('error')
41 '%s returned error: %s - %s' % (self
.IE_NAME
, error
, response
.get('message')),
44 def _download_json(self
, url
, video_id
, note
='Downloading JSON metadata'):
46 'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
47 'X-Requested-With': 'XMLHttpRequest',
49 for cookie
in self
._downloader
.cookiejar
:
50 if cookie
.name
== 'api_token':
51 headers
['Twitch-Api-Token'] = cookie
.value
52 request
= sanitized_Request(url
, headers
=headers
)
53 response
= super(TwitchBaseIE
, self
)._download
_json
(request
, video_id
, note
)
54 self
._handle
_error
(response
)
57 def _real_initialize(self
):
61 (username
, password
) = self
._get
_login
_info
()
65 login_page
, handle
= self
._download
_webpage
_handle
(
66 self
._LOGIN
_URL
, None, 'Downloading login page')
68 login_form
= self
._hidden
_inputs
(login_page
)
75 redirect_url
= handle
.geturl()
77 post_url
= self
._search
_regex
(
78 r
'<form[^>]+action=(["\'])(?P
<url
>.+?
)\
1', login_page,
79 'post url
', default=redirect_url, group='url
')
81 if not post_url.startswith('http
'):
82 post_url = compat_urlparse.urljoin(redirect_url, post_url)
84 request = sanitized_Request(
85 post_url, urlencode_postdata(login_form))
86 request.add_header('Referer
', redirect_url)
87 response = self._download_webpage(
88 request, None, 'Logging
in as %s' % username)
90 error_message = self._search_regex(
91 r'<div
[^
>]+class="subwindow_notice"[^
>]*>([^
<]+)</div
>',
92 response, 'error message
', default=None)
95 'Unable to login
. Twitch said
: %s' % error_message, expected=True)
97 if '>Reset your password
<' in response:
98 self.report_warning('Twitch asks you to reset your password
, go to https
://secure
.twitch
.tv
/reset
/submit
')
100 def _prefer_source(self, formats):
102 source = next(f for f in formats if f['format_id
'] == 'Source
')
103 source['preference
'] = 10
104 except StopIteration:
105 pass # No Source stream present
106 self._sort_formats(formats)
109 class TwitchItemBaseIE(TwitchBaseIE):
110 def _download_info(self, item, item_id):
111 return self._extract_info(self._download_json(
112 '%s/kraken
/videos
/%s%s' % (self._API_BASE, item, item_id), item_id,
113 'Downloading
%s info JSON
' % self._ITEM_TYPE))
115 def _extract_media(self, item_id):
116 info = self._download_info(self._ITEM_SHORTCUT, item_id)
117 response = self._download_json(
118 '%s/api
/videos
/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
119 'Downloading
%s playlist JSON
' % self._ITEM_TYPE)
121 chunks = response['chunks
']
122 qualities = list(chunks.keys())
123 for num, fragment in enumerate(zip(*chunks.values()), start=1):
125 for fmt_num, fragment_fmt in enumerate(fragment):
126 format_id = qualities[fmt_num]
128 'url
': fragment_fmt['url
'],
129 'format_id
': format_id,
130 'quality
': 1 if format_id == 'live
' else 0,
132 m = re.search(r'^
(?P
<height
>\d
+)[Pp
]', format_id)
134 fmt['height
'] = int(m.group('height
'))
136 self._sort_formats(formats)
138 entry['id'] = '%s_%d' % (entry['id'], num)
139 entry['title
'] = '%s part
%d' % (entry['title
'], num)
140 entry['formats
'] = formats
141 entries.append(entry)
142 return self.playlist_result(entries, info['id'], info['title
'])
144 def _extract_info(self, info):
147 'title
': info.get('title
') or 'Untitled Broadcast
',
148 'description
': info.get('description
'),
149 'duration
': int_or_none(info.get('length
')),
150 'thumbnail
': info.get('preview
'),
151 'uploader
': info.get('channel
', {}).get('display_name
'),
152 'uploader_id
': info.get('channel
', {}).get('name
'),
153 'timestamp
': parse_iso8601(info.get('recorded_at
')),
154 'view_count
': int_or_none(info.get('views
')),
157 def _real_extract(self, url):
158 return self._extract_media(self._match_id(url))
161 class TwitchVideoIE(TwitchItemBaseIE):
162 IE_NAME = 'twitch
:video
'
163 _VALID_URL = r'%s/[^
/]+/b
/(?P
<id>\d
+)' % TwitchBaseIE._VALID_URL_BASE
168 'url
': 'http
://www
.twitch
.tv
/riotgames
/b
/577357806',
171 'title
': 'Worlds Semifinals
- Star Horn Royal Club vs
. OMG
',
173 'playlist_mincount
': 12,
174 'skip
': 'HTTP Error
404: Not Found
',
178 class TwitchChapterIE(TwitchItemBaseIE):
179 IE_NAME = 'twitch
:chapter
'
180 _VALID_URL = r'%s/[^
/]+/c
/(?P
<id>\d
+)' % TwitchBaseIE._VALID_URL_BASE
181 _ITEM_TYPE = 'chapter
'
185 'url
': 'http
://www
.twitch
.tv
/acracingleague
/c
/5285812',
188 'title
': 'ACRL Off Season
- Sports Cars
@ Nordschleife
',
190 'playlist_mincount
': 3,
191 'skip
': 'HTTP Error
404: Not Found
',
193 'url
': 'http
://www
.twitch
.tv
/tsm_theoddone
/c
/2349361',
194 'only_matching
': True,
198 class TwitchVodIE(TwitchItemBaseIE):
199 IE_NAME = 'twitch
:vod
'
200 _VALID_URL = r'%s/[^
/]+/v
/(?P
<id>\d
+)' % TwitchBaseIE._VALID_URL_BASE
205 'url
': 'http
://www
.twitch
.tv
/riotgames
/v
/6528877?t
=5m10s
',
209 'title
': 'LCK Summer Split
- Week
6 Day
1',
210 'thumbnail
': 're
:^https?
://.*\
.jpg$
',
212 'timestamp
': 1435131709,
213 'upload_date
': '20150624',
214 'uploader
': 'Riot Games
',
215 'uploader_id
': 'riotgames
',
221 'skip_download
': True,
224 # Untitled broadcast (title is None)
225 'url
': 'http
://www
.twitch
.tv
/belkao_o
/v
/11230755',
229 'title
': 'Untitled Broadcast
',
230 'thumbnail
': 're
:^https?
://.*\
.jpg$
',
232 'timestamp
': 1439746708,
233 'upload_date
': '20150816',
234 'uploader
': 'BelkAO_o
',
235 'uploader_id
': 'belkao_o
',
240 'skip_download
': True,
244 def _real_extract(self, url):
245 item_id = self._match_id(url)
247 info = self._download_info(self._ITEM_SHORTCUT, item_id)
248 access_token = self._download_json(
249 '%s/api
/vods
/%s/access_token
' % (self._API_BASE, item_id), item_id,
250 'Downloading
%s access token
' % self._ITEM_TYPE)
252 formats = self._extract_m3u8_formats(
254 self._USHER_BASE, item_id,
255 compat_urllib_parse_urlencode({
256 'allow_source
': 'true
',
257 'allow_audio_only
': 'true
',
258 'allow_spectre
': 'true
',
259 'player
': 'twitchweb
',
260 'nauth
': access_token['token
'],
261 'nauthsig
': access_token['sig
'],
263 item_id, 'mp4
', entry_protocol='m3u8_native
')
265 self._prefer_source(formats)
266 info['formats
'] = formats
268 parsed_url = compat_urllib_parse_urlparse(url)
269 query = compat_parse_qs(parsed_url.query)
271 info['start_time
'] = parse_duration(query['t
'][0])
276 class TwitchPlaylistBaseIE(TwitchBaseIE):
277 _PLAYLIST_URL = '%s/kraken
/channels
/%%s/videos
/?offset
=%%d&limit
=%%d' % TwitchBaseIE._API_BASE
280 def _extract_playlist(self, channel_id):
281 info = self._download_json(
282 '%s/kraken
/channels
/%s' % (self._API_BASE, channel_id),
283 channel_id, 'Downloading channel info JSON
')
284 channel_name = info.get('display_name
') or info.get('name
')
287 limit = self._PAGE_LIMIT
288 broken_paging_detected = False
289 counter_override = None
290 for counter in itertools.count(1):
291 response = self._download_json(
292 self._PLAYLIST_URL % (channel_id, offset, limit),
294 'Downloading
%s videos JSON page
%s'
295 % (self._PLAYLIST_TYPE, counter_override or counter))
296 page_entries = self._extract_playlist_page(response)
299 total = int_or_none(response.get('_total
'))
300 # Since the beginning of March 2016 twitch's paging mechanism
301 # is completely broken on the twitch side. It simply ignores
302 # a limit and returns the whole offset number of videos.
303 # Working around by just requesting all videos at once.
304 # Upd: pagination bug was fixed by twitch on 15.03.2016.
305 if not broken_paging_detected
and total
and len(page_entries
) > limit
:
307 'Twitch pagination is broken on twitch side, requesting all videos at once',
309 broken_paging_detected
= True
311 counter_override
= '(all at once)'
313 entries
.extend(page_entries
)
314 if broken_paging_detected
or total
and len(page_entries
) >= total
:
317 return self
.playlist_result(
318 [self
.url_result(entry
) for entry
in orderedSet(entries
)],
319 channel_id
, channel_name
)
321 def _extract_playlist_page(self
, response
):
322 videos
= response
.get('videos')
323 return [video
['url'] for video
in videos
] if videos
else []
325 def _real_extract(self
, url
):
326 return self
._extract
_playlist
(self
._match
_id
(url
))
329 class TwitchProfileIE(TwitchPlaylistBaseIE
):
330 IE_NAME
= 'twitch:profile'
331 _VALID_URL
= r
'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE
._VALID
_URL
_BASE
332 _PLAYLIST_TYPE
= 'profile'
335 'url': 'http://www.twitch.tv/vanillatv/profile',
338 'title': 'VanillaTV',
340 'playlist_mincount': 412,
344 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE
):
345 IE_NAME
= 'twitch:past_broadcasts'
346 _VALID_URL
= r
'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE
._VALID
_URL
_BASE
347 _PLAYLIST_URL
= TwitchPlaylistBaseIE
._PLAYLIST
_URL
+ '&broadcasts=true'
348 _PLAYLIST_TYPE
= 'past broadcasts'
351 'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
356 'playlist_mincount': 54,
360 class TwitchStreamIE(TwitchBaseIE
):
361 IE_NAME
= 'twitch:stream'
362 _VALID_URL
= r
'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE
._VALID
_URL
_BASE
365 'url': 'http://www.twitch.tv/shroomztv',
368 'display_id': 'shroomztv',
370 'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
371 'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
373 'timestamp': 1421928037,
374 'upload_date': '20150122',
375 'uploader': 'ShroomzTV',
376 'uploader_id': 'shroomztv',
381 'skip_download': True,
384 'url': 'http://www.twitch.tv/miracle_doto#profile-0',
385 'only_matching': True,
388 def _real_extract(self
, url
):
389 channel_id
= self
._match
_id
(url
)
391 stream
= self
._download
_json
(
392 '%s/kraken/streams/%s' % (self
._API
_BASE
, channel_id
), channel_id
,
393 'Downloading stream JSON').get('stream')
395 # Fallback on profile extraction if stream is offline
397 return self
.url_result(
398 'http://www.twitch.tv/%s/profile' % channel_id
,
399 'TwitchProfile', channel_id
)
401 # Channel name may be typed if different case than the original channel name
402 # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
403 # an invalid m3u8 URL. Working around by use of original channel name from stream
404 # JSON and fallback to lowercase if it's not available.
405 channel_id
= stream
.get('channel', {}).get('name') or channel_id
.lower()
407 access_token
= self
._download
_json
(
408 '%s/api/channels/%s/access_token' % (self
._API
_BASE
, channel_id
), channel_id
,
409 'Downloading channel access token')
412 'allow_source': 'true',
413 'allow_audio_only': 'true',
414 'p': random
.randint(1000000, 10000000),
415 'player': 'twitchweb',
416 'segment_preference': '4',
417 'sig': access_token
['sig'].encode('utf-8'),
418 'token': access_token
['token'].encode('utf-8'),
420 formats
= self
._extract
_m
3u8_formats
(
421 '%s/api/channel/hls/%s.m3u8?%s'
422 % (self
._USHER
_BASE
, channel_id
, compat_urllib_parse_urlencode(query
)),
424 self
._prefer
_source
(formats
)
426 view_count
= stream
.get('viewers')
427 timestamp
= parse_iso8601(stream
.get('created_at'))
429 channel
= stream
['channel']
430 title
= self
._live
_title
(channel
.get('display_name') or channel
.get('name'))
431 description
= channel
.get('status')
434 for thumbnail_key
, thumbnail_url
in stream
['preview'].items():
435 m
= re
.search(r
'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key
)
439 'url': thumbnail_url
,
440 'width': int(m
.group('width')),
441 'height': int(m
.group('height')),
445 'id': compat_str(stream
['_id']),
446 'display_id': channel_id
,
448 'description': description
,
449 'thumbnails': thumbnails
,
450 'uploader': channel
.get('display_name'),
451 'uploader_id': channel
.get('name'),
452 'timestamp': timestamp
,
453 'view_count': view_count
,