yt_dlp/extractor/twitch.py

   1 import collections
   2 import itertools
   3 import json
   4 import random
   5 import re
   6 import urllib.parse
   7
   8 from .common import InfoExtractor
   9 from ..utils import (
  10     ExtractorError,
  11     UserNotLive,
  12     base_url,
  13     clean_html,
  14     dict_get,
  15     float_or_none,
  16     int_or_none,
  17     make_archive_id,
  18     parse_duration,
  19     parse_iso8601,
  20     parse_qs,
  21     qualities,
  22     str_or_none,
  23     traverse_obj,
  24     try_get,
  25     unified_timestamp,
  26     update_url_query,
  27     url_or_none,
  28     urljoin,
  29 )
  30
  31
  32 class TwitchBaseIE(InfoExtractor):
  33     _VALID_URL_BASE = r'https?://(?:(?:www|go|m)\.)?twitch\.tv'
  34
  35     _API_BASE = 'https://api.twitch.tv'
  36     _USHER_BASE = 'https://usher.ttvnw.net'
  37     _LOGIN_FORM_URL = 'https://www.twitch.tv/login'
  38     _LOGIN_POST_URL = 'https://passport.twitch.tv/login'
  39     _NETRC_MACHINE = 'twitch'
  40
  41     _OPERATION_HASHES = {
  42         'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
  43         'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
  44         'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
  45         'ChannelCollectionsContent': '447aec6a0cc1e8d0a8d7732d47eb0762c336a2294fdb009e9c9d854e49d484b9',
  46         'StreamMetadata': 'a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962',
  47         'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
  48         'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
  49         'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
  50         'VideoMetadata': '49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad',
  51         'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
  52         'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
  53     }
  54
  55     @property
  56     def _CLIENT_ID(self):
  57         return self._configuration_arg(
  58             'client_id', ['ue6666qo983tsx6so1t0vnawi233wa'], ie_key='Twitch', casesense=True)[0]
  59
  60     def _perform_login(self, username, password):
  61         def fail(message):
  62             raise ExtractorError(
  63                 f'Unable to login. Twitch said: {message}', expected=True)
  64
  65         def login_step(page, urlh, note, data):
  66             form = self._hidden_inputs(page)
  67             form.update(data)
  68
  69             page_url = urlh.url
  70             post_url = self._search_regex(
  71                 r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page,
  72                 'post url', default=self._LOGIN_POST_URL, group='url')
  73             post_url = urljoin(page_url, post_url)
  74
  75             headers = {
  76                 'Referer': page_url,
  77                 'Origin': 'https://www.twitch.tv',
  78                 'Content-Type': 'text/plain;charset=UTF-8',
  79             }
  80
  81             response = self._download_json(
  82                 post_url, None, note, data=json.dumps(form).encode(),
  83                 headers=headers, expected_status=400)
  84             error = dict_get(response, ('error', 'error_description', 'error_code'))
  85             if error:
  86                 fail(error)
  87
  88             if 'Authenticated successfully' in response.get('message', ''):
  89                 return None, None
  90
  91             redirect_url = urljoin(
  92                 post_url,
  93                 response.get('redirect') or response['redirect_path'])
  94             return self._download_webpage_handle(
  95                 redirect_url, None, 'Downloading login redirect page',
  96                 headers=headers)
  97
  98         login_page, handle = self._download_webpage_handle(
  99             self._LOGIN_FORM_URL, None, 'Downloading login page')
 100
 101         # Some TOR nodes and public proxies are blocked completely
 102         if 'blacklist_message' in login_page:
 103             fail(clean_html(login_page))
 104
 105         redirect_page, handle = login_step(
 106             login_page, handle, 'Logging in', {
 107                 'username': username,
 108                 'password': password,
 109                 'client_id': self._CLIENT_ID,
 110             })
 111
 112         # Successful login
 113         if not redirect_page:
 114             return
 115
 116         if re.search(r'(?i)<form[^>]+id="two-factor-submit"', redirect_page) is not None:
 117             # TODO: Add mechanism to request an SMS or phone call
 118             tfa_token = self._get_tfa_info('two-factor authentication token')
 119             login_step(redirect_page, handle, 'Submitting TFA token', {
 120                 'authy_token': tfa_token,
 121                 'remember_2fa': 'true',
 122             })
 123
 124     def _prefer_source(self, formats):
 125         try:
 126             source = next(f for f in formats if f['format_id'] == 'Source')
 127             source['quality'] = 10
 128         except StopIteration:
 129             for f in formats:
 130                 if '/chunked/' in f['url']:
 131                     f.update({
 132                         'quality': 10,
 133                         'format_note': 'Source',
 134                     })
 135
 136     def _download_base_gql(self, video_id, ops, note, fatal=True):
 137         headers = {
 138             'Content-Type': 'text/plain;charset=UTF-8',
 139             'Client-ID': self._CLIENT_ID,
 140         }
 141         gql_auth = self._get_cookies('https://gql.twitch.tv').get('auth-token')
 142         if gql_auth:
 143             headers['Authorization'] = 'OAuth ' + gql_auth.value
 144         return self._download_json(
 145             'https://gql.twitch.tv/gql', video_id, note,
 146             data=json.dumps(ops).encode(),
 147             headers=headers, fatal=fatal)
 148
 149     def _download_gql(self, video_id, ops, note, fatal=True):
 150         for op in ops:
 151             op['extensions'] = {
 152                 'persistedQuery': {
 153                     'version': 1,
 154                     'sha256Hash': self._OPERATION_HASHES[op['operationName']],
 155                 },
 156             }
 157         return self._download_base_gql(video_id, ops, note)
 158
 159     def _download_access_token(self, video_id, token_kind, param_name):
 160         method = f'{token_kind}PlaybackAccessToken'
 161         ops = {
 162             'query': '''{
 163               %s(
 164                 %s: "%s",
 165                 params: {
 166                   platform: "web",
 167                   playerBackend: "mediaplayer",
 168                   playerType: "site"
 169                 }
 170               )
 171               {
 172                 value
 173                 signature
 174               }
 175             }''' % (method, param_name, video_id),  # noqa: UP031
 176         }
 177         return self._download_base_gql(
 178             video_id, ops,
 179             f'Downloading {token_kind} access token GraphQL')['data'][method]
 180
 181     def _get_thumbnails(self, thumbnail):
 182         return [{
 183             'url': re.sub(r'\d+x\d+(\.\w+)($|(?=[?#]))', r'0x0\g<1>', thumbnail),
 184             'preference': 1,
 185         }, {
 186             'url': thumbnail,
 187         }] if thumbnail else None
 188
 189     def _extract_twitch_m3u8_formats(self, path, video_id, token, signature):
 190         formats = self._extract_m3u8_formats(
 191             f'{self._USHER_BASE}/{path}/{video_id}.m3u8', video_id, 'mp4', query={
 192                 'allow_source': 'true',
 193                 'allow_audio_only': 'true',
 194                 'allow_spectre': 'true',
 195                 'p': random.randint(1000000, 10000000),
 196                 'platform': 'web',
 197                 'player': 'twitchweb',
 198                 'supported_codecs': 'av1,h265,h264',
 199                 'playlist_include_framerate': 'true',
 200                 'sig': signature,
 201                 'token': token,
 202             })
 203         for fmt in formats:
 204             if fmt.get('vcodec') and fmt['vcodec'].startswith('av01'):
 205                 # mpegts does not yet have proper support for av1
 206                 fmt['downloader_options'] = {'ffmpeg_args_out': ['-f', 'mp4']}
 207
 208         return formats
 209
 210
 211 class TwitchVodIE(TwitchBaseIE):
 212     IE_NAME = 'twitch:vod'
 213     _VALID_URL = r'''(?x)
 214                     https?://
 215                         (?:
 216                             (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/v(?:ideo)?|videos)/|
 217                             player\.twitch\.tv/\?.*?\bvideo=v?|
 218                             www\.twitch\.tv/[^/]+/schedule\?vodID=
 219                         )
 220                         (?P<id>\d+)
 221                     '''
 222
 223     _TESTS = [{
 224         'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
 225         'info_dict': {
 226             'id': 'v6528877',
 227             'ext': 'mp4',
 228             'title': 'LCK Summer Split - Week 6 Day 1',
 229             'thumbnail': r're:^https?://.*\.jpg$',
 230             'duration': 17208,
 231             'timestamp': 1435131734,
 232             'upload_date': '20150624',
 233             'uploader': 'Riot Games',
 234             'uploader_id': 'riotgames',
 235             'view_count': int,
 236             'start_time': 310,
 237             'chapters': [
 238                 {
 239                     'start_time': 0,
 240                     'end_time': 17208,
 241                     'title': 'League of Legends',
 242                 },
 243             ],
 244             'live_status': 'was_live',
 245         },
 246         'params': {
 247             # m3u8 download
 248             'skip_download': True,
 249         },
 250     }, {
 251         # Untitled broadcast (title is None)
 252         'url': 'http://www.twitch.tv/belkao_o/v/11230755',
 253         'info_dict': {
 254             'id': 'v11230755',
 255             'ext': 'mp4',
 256             'title': 'Untitled Broadcast',
 257             'thumbnail': r're:^https?://.*\.jpg$',
 258             'duration': 1638,
 259             'timestamp': 1439746708,
 260             'upload_date': '20150816',
 261             'uploader': 'BelkAO_o',
 262             'uploader_id': 'belkao_o',
 263             'view_count': int,
 264         },
 265         'params': {
 266             # m3u8 download
 267             'skip_download': True,
 268         },
 269         'skip': 'HTTP Error 404: Not Found',
 270     }, {
 271         'url': 'http://player.twitch.tv/?t=5m10s&video=v6528877',
 272         'only_matching': True,
 273     }, {
 274         'url': 'https://www.twitch.tv/videos/6528877',
 275         'only_matching': True,
 276     }, {
 277         'url': 'https://m.twitch.tv/beagsandjam/v/247478721',
 278         'only_matching': True,
 279     }, {
 280         'url': 'https://www.twitch.tv/northernlion/video/291940395',
 281         'only_matching': True,
 282     }, {
 283         'url': 'https://player.twitch.tv/?video=480452374',
 284         'only_matching': True,
 285     }, {
 286         'url': 'https://www.twitch.tv/videos/635475444',
 287         'info_dict': {
 288             'id': 'v635475444',
 289             'ext': 'mp4',
 290             'title': 'Riot Games',
 291             'duration': 11643,
 292             'uploader': 'Riot Games',
 293             'uploader_id': 'riotgames',
 294             'timestamp': 1590770569,
 295             'upload_date': '20200529',
 296             'chapters': [
 297                 {
 298                     'start_time': 0,
 299                     'end_time': 573,
 300                     'title': 'League of Legends',
 301                 },
 302                 {
 303                     'start_time': 573,
 304                     'end_time': 3922,
 305                     'title': 'Legends of Runeterra',
 306                 },
 307                 {
 308                     'start_time': 3922,
 309                     'end_time': 11643,
 310                     'title': 'Art',
 311                 },
 312             ],
 313             'live_status': 'was_live',
 314             'thumbnail': r're:^https?://.*\.jpg$',
 315             'view_count': int,
 316         },
 317         'params': {
 318             'skip_download': True,
 319         },
 320     }, {
 321         'note': 'Storyboards',
 322         'url': 'https://www.twitch.tv/videos/635475444',
 323         'info_dict': {
 324             'id': 'v635475444',
 325             'format_id': 'sb0',
 326             'ext': 'mhtml',
 327             'title': 'Riot Games',
 328             'duration': 11643,
 329             'uploader': 'Riot Games',
 330             'uploader_id': 'riotgames',
 331             'timestamp': 1590770569,
 332             'upload_date': '20200529',
 333             'chapters': [
 334                 {
 335                     'start_time': 0,
 336                     'end_time': 573,
 337                     'title': 'League of Legends',
 338                 },
 339                 {
 340                     'start_time': 573,
 341                     'end_time': 3922,
 342                     'title': 'Legends of Runeterra',
 343                 },
 344                 {
 345                     'start_time': 3922,
 346                     'end_time': 11643,
 347                     'title': 'Art',
 348                 },
 349             ],
 350             'live_status': 'was_live',
 351             'thumbnail': r're:^https?://.*\.jpg$',
 352             'view_count': int,
 353             'columns': int,
 354             'rows': int,
 355         },
 356         'params': {
 357             'format': 'mhtml',
 358             'skip_download': True,
 359         },
 360     }, {
 361         'note': 'VOD with single chapter',
 362         'url': 'https://www.twitch.tv/videos/1536751224',
 363         'info_dict': {
 364             'id': 'v1536751224',
 365             'ext': 'mp4',
 366             'title': 'Porter Robinson Star Guardian Stream Tour with LilyPichu',
 367             'duration': 8353,
 368             'uploader': 'Riot Games',
 369             'uploader_id': 'riotgames',
 370             'timestamp': 1658267731,
 371             'upload_date': '20220719',
 372             'chapters': [
 373                 {
 374                     'start_time': 0,
 375                     'end_time': 8353,
 376                     'title': 'League of Legends',
 377                 },
 378             ],
 379             'live_status': 'was_live',
 380             'thumbnail': r're:^https?://.*\.jpg$',
 381             'view_count': int,
 382         },
 383         'params': {
 384             'skip_download': True,
 385         },
 386         'expected_warnings': ['Unable to download JSON metadata: HTTP Error 403: Forbidden'],
 387     }, {
 388         'url': 'https://www.twitch.tv/tangotek/schedule?vodID=1822395420',
 389         'only_matching': True,
 390     }]
 391
 392     def _download_info(self, item_id):
 393         data = self._download_gql(
 394             item_id, [{
 395                 'operationName': 'VideoMetadata',
 396                 'variables': {
 397                     'channelLogin': '',
 398                     'videoID': item_id,
 399                 },
 400             }, {
 401                 'operationName': 'VideoPlayer_ChapterSelectButtonVideo',
 402                 'variables': {
 403                     'includePrivate': False,
 404                     'videoID': item_id,
 405                 },
 406             }, {
 407                 'operationName': 'VideoPlayer_VODSeekbarPreviewVideo',
 408                 'variables': {
 409                     'includePrivate': False,
 410                     'videoID': item_id,
 411                 },
 412             }],
 413             'Downloading stream metadata GraphQL')
 414
 415         video = traverse_obj(data, (..., 'data', 'video'), get_all=False)
 416         if video is None:
 417             raise ExtractorError(f'Video {item_id} does not exist', expected=True)
 418
 419         video['moments'] = traverse_obj(data, (..., 'data', 'video', 'moments', 'edges', ..., 'node'))
 420         video['storyboard'] = traverse_obj(
 421             data, (..., 'data', 'video', 'seekPreviewsURL', {url_or_none}), get_all=False)
 422
 423         return video
 424
 425     def _extract_info(self, info):
 426         status = info.get('status')
 427         if status == 'recording':
 428             is_live = True
 429         elif status == 'recorded':
 430             is_live = False
 431         else:
 432             is_live = None
 433         _QUALITIES = ('small', 'medium', 'large')
 434         quality_key = qualities(_QUALITIES)
 435         thumbnails = []
 436         preview = info.get('preview')
 437         if isinstance(preview, dict):
 438             for thumbnail_id, thumbnail_url in preview.items():
 439                 thumbnail_url = url_or_none(thumbnail_url)
 440                 if not thumbnail_url:
 441                     continue
 442                 if thumbnail_id not in _QUALITIES:
 443                     continue
 444                 thumbnails.append({
 445                     'url': thumbnail_url,
 446                     'preference': quality_key(thumbnail_id),
 447                 })
 448         return {
 449             'id': info['_id'],
 450             'title': info.get('title') or 'Untitled Broadcast',
 451             'description': info.get('description'),
 452             'duration': int_or_none(info.get('length')),
 453             'thumbnails': thumbnails,
 454             'uploader': info.get('channel', {}).get('display_name'),
 455             'uploader_id': info.get('channel', {}).get('name'),
 456             'timestamp': parse_iso8601(info.get('recorded_at')),
 457             'view_count': int_or_none(info.get('views')),
 458             'is_live': is_live,
 459             'was_live': True,
 460         }
 461
 462     def _extract_chapters(self, info, item_id):
 463         if not info.get('moments'):
 464             game = traverse_obj(info, ('game', 'displayName'))
 465             if game:
 466                 yield {'title': game}
 467             return
 468
 469         for moment in info['moments']:
 470             start_time = int_or_none(moment.get('positionMilliseconds'), 1000)
 471             duration = int_or_none(moment.get('durationMilliseconds'), 1000)
 472             name = str_or_none(moment.get('description'))
 473
 474             if start_time is None or duration is None:
 475                 self.report_warning(f'Important chapter information missing for chapter {name}', item_id)
 476                 continue
 477             yield {
 478                 'start_time': start_time,
 479                 'end_time': start_time + duration,
 480                 'title': name,
 481             }
 482
 483     def _extract_info_gql(self, info, item_id):
 484         vod_id = info.get('id') or item_id
 485         # id backward compatibility for download archives
 486         if vod_id[0] != 'v':
 487             vod_id = f'v{vod_id}'
 488         thumbnail = url_or_none(info.get('previewThumbnailURL'))
 489         is_live = None
 490         if thumbnail:
 491             if re.findall(r'/404_processing_[^.?#]+\.png', thumbnail):
 492                 is_live, thumbnail = True, None
 493             else:
 494                 is_live = False
 495
 496         return {
 497             'id': vod_id,
 498             'title': info.get('title') or 'Untitled Broadcast',
 499             'description': info.get('description'),
 500             'duration': int_or_none(info.get('lengthSeconds')),
 501             'thumbnails': self._get_thumbnails(thumbnail),
 502             'uploader': try_get(info, lambda x: x['owner']['displayName'], str),
 503             'uploader_id': try_get(info, lambda x: x['owner']['login'], str),
 504             'timestamp': unified_timestamp(info.get('publishedAt')),
 505             'view_count': int_or_none(info.get('viewCount')),
 506             'chapters': list(self._extract_chapters(info, item_id)),
 507             'is_live': is_live,
 508             'was_live': True,
 509         }
 510
 511     def _extract_storyboard(self, item_id, storyboard_json_url, duration):
 512         if not duration or not storyboard_json_url:
 513             return
 514         spec = self._download_json(storyboard_json_url, item_id, 'Downloading storyboard metadata JSON', fatal=False) or []
 515         # sort from highest quality to lowest
 516         # This makes sb0 the highest-quality format, sb1 - lower, etc which is consistent with youtube sb ordering
 517         spec.sort(key=lambda x: int_or_none(x.get('width')) or 0, reverse=True)
 518         base = base_url(storyboard_json_url)
 519         for i, s in enumerate(spec):
 520             count = int_or_none(s.get('count'))
 521             images = s.get('images')
 522             if not (images and count):
 523                 continue
 524             fragment_duration = duration / len(images)
 525             yield {
 526                 'format_id': f'sb{i}',
 527                 'format_note': 'storyboard',
 528                 'ext': 'mhtml',
 529                 'protocol': 'mhtml',
 530                 'acodec': 'none',
 531                 'vcodec': 'none',
 532                 'url': urljoin(base, images[0]),
 533                 'width': int_or_none(s.get('width')),
 534                 'height': int_or_none(s.get('height')),
 535                 'fps': count / duration,
 536                 'rows': int_or_none(s.get('rows')),
 537                 'columns': int_or_none(s.get('cols')),
 538                 'fragments': [{
 539                     'url': urljoin(base, path),
 540                     'duration': fragment_duration,
 541                 } for path in images],
 542             }
 543
 544     def _real_extract(self, url):
 545         vod_id = self._match_id(url)
 546
 547         video = self._download_info(vod_id)
 548         info = self._extract_info_gql(video, vod_id)
 549         access_token = self._download_access_token(vod_id, 'video', 'id')
 550
 551         formats = self._extract_twitch_m3u8_formats(
 552             'vod', vod_id, access_token['value'], access_token['signature'])
 553         formats.extend(self._extract_storyboard(vod_id, video.get('storyboard'), info.get('duration')))
 554
 555         self._prefer_source(formats)
 556         info['formats'] = formats
 557
 558         parsed_url = urllib.parse.urlparse(url)
 559         query = urllib.parse.parse_qs(parsed_url.query)
 560         if 't' in query:
 561             info['start_time'] = parse_duration(query['t'][0])
 562
 563         if info.get('timestamp') is not None:
 564             info['subtitles'] = {
 565                 'rechat': [{
 566                     'url': update_url_query(
 567                         f'https://api.twitch.tv/v5/videos/{vod_id}/comments', {
 568                             'client_id': self._CLIENT_ID,
 569                         }),
 570                     'ext': 'json',
 571                 }],
 572             }
 573
 574         return info
 575
 576
 577 def _make_video_result(node):
 578     assert isinstance(node, dict)
 579     video_id = node.get('id')
 580     if not video_id:
 581         return
 582     return {
 583         '_type': 'url_transparent',
 584         'ie_key': TwitchVodIE.ie_key(),
 585         'id': 'v' + video_id,
 586         'url': f'https://www.twitch.tv/videos/{video_id}',
 587         'title': node.get('title'),
 588         'thumbnail': node.get('previewThumbnailURL'),
 589         'duration': float_or_none(node.get('lengthSeconds')),
 590         'view_count': int_or_none(node.get('viewCount')),
 591     }
 592
 593
 594 class TwitchCollectionIE(TwitchBaseIE):
 595     _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'
 596
 597     _TESTS = [{
 598         'url': 'https://www.twitch.tv/collections/wlDCoH0zEBZZbQ',
 599         'info_dict': {
 600             'id': 'wlDCoH0zEBZZbQ',
 601             'title': 'Overthrow Nook, capitalism for children',
 602         },
 603         'playlist_mincount': 13,
 604     }]
 605
 606     _OPERATION_NAME = 'CollectionSideBar'
 607
 608     def _real_extract(self, url):
 609         collection_id = self._match_id(url)
 610         collection = self._download_gql(
 611             collection_id, [{
 612                 'operationName': self._OPERATION_NAME,
 613                 'variables': {'collectionID': collection_id},
 614             }],
 615             'Downloading collection GraphQL')[0]['data']['collection']
 616         title = collection.get('title')
 617         entries = []
 618         for edge in collection['items']['edges']:
 619             if not isinstance(edge, dict):
 620                 continue
 621             node = edge.get('node')
 622             if not isinstance(node, dict):
 623                 continue
 624             video = _make_video_result(node)
 625             if video:
 626                 entries.append(video)
 627         return self.playlist_result(
 628             entries, playlist_id=collection_id, playlist_title=title)
 629
 630
 631 class TwitchPlaylistBaseIE(TwitchBaseIE):
 632     _PAGE_LIMIT = 100
 633
 634     def _entries(self, channel_name, *args):
 635         cursor = None
 636         variables_common = self._make_variables(channel_name, *args)
 637         entries_key = f'{self._ENTRY_KIND}s'
 638         for page_num in itertools.count(1):
 639             variables = variables_common.copy()
 640             variables['limit'] = self._PAGE_LIMIT
 641             if cursor:
 642                 variables['cursor'] = cursor
 643             page = self._download_gql(
 644                 channel_name, [{
 645                     'operationName': self._OPERATION_NAME,
 646                     'variables': variables,
 647                 }],
 648                 f'Downloading {self._NODE_KIND}s GraphQL page {page_num}',
 649                 fatal=False)
 650             if not page:
 651                 break
 652             edges = try_get(
 653                 page, lambda x: x[0]['data']['user'][entries_key]['edges'], list)
 654             if not edges:
 655                 break
 656             for edge in edges:
 657                 if not isinstance(edge, dict):
 658                     continue
 659                 if edge.get('__typename') != self._EDGE_KIND:
 660                     continue
 661                 node = edge.get('node')
 662                 if not isinstance(node, dict):
 663                     continue
 664                 if node.get('__typename') != self._NODE_KIND:
 665                     continue
 666                 entry = self._extract_entry(node)
 667                 if entry:
 668                     cursor = edge.get('cursor')
 669                     yield entry
 670             if not cursor or not isinstance(cursor, str):
 671                 break
 672
 673
 674 class TwitchVideosIE(TwitchPlaylistBaseIE):
 675     _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'
 676
 677     _TESTS = [{
 678         # All Videos sorted by Date
 679         'url': 'https://www.twitch.tv/spamfish/videos?filter=all',
 680         'info_dict': {
 681             'id': 'spamfish',
 682             'title': 'spamfish - All Videos sorted by Date',
 683         },
 684         'playlist_mincount': 924,
 685     }, {
 686         # All Videos sorted by Popular
 687         'url': 'https://www.twitch.tv/spamfish/videos?filter=all&sort=views',
 688         'info_dict': {
 689             'id': 'spamfish',
 690             'title': 'spamfish - All Videos sorted by Popular',
 691         },
 692         'playlist_mincount': 931,
 693     }, {
 694         # Past Broadcasts sorted by Date
 695         'url': 'https://www.twitch.tv/spamfish/videos?filter=archives',
 696         'info_dict': {
 697             'id': 'spamfish',
 698             'title': 'spamfish - Past Broadcasts sorted by Date',
 699         },
 700         'playlist_mincount': 27,
 701     }, {
 702         # Highlights sorted by Date
 703         'url': 'https://www.twitch.tv/spamfish/videos?filter=highlights',
 704         'info_dict': {
 705             'id': 'spamfish',
 706             'title': 'spamfish - Highlights sorted by Date',
 707         },
 708         'playlist_mincount': 901,
 709     }, {
 710         # Uploads sorted by Date
 711         'url': 'https://www.twitch.tv/esl_csgo/videos?filter=uploads&sort=time',
 712         'info_dict': {
 713             'id': 'esl_csgo',
 714             'title': 'esl_csgo - Uploads sorted by Date',
 715         },
 716         'playlist_mincount': 5,
 717     }, {
 718         # Past Premieres sorted by Date
 719         'url': 'https://www.twitch.tv/spamfish/videos?filter=past_premieres',
 720         'info_dict': {
 721             'id': 'spamfish',
 722             'title': 'spamfish - Past Premieres sorted by Date',
 723         },
 724         'playlist_mincount': 1,
 725     }, {
 726         'url': 'https://www.twitch.tv/spamfish/videos/all',
 727         'only_matching': True,
 728     }, {
 729         'url': 'https://m.twitch.tv/spamfish/videos/all',
 730         'only_matching': True,
 731     }, {
 732         'url': 'https://www.twitch.tv/spamfish/videos',
 733         'only_matching': True,
 734     }]
 735
 736     Broadcast = collections.namedtuple('Broadcast', ['type', 'label'])
 737
 738     _DEFAULT_BROADCAST = Broadcast(None, 'All Videos')
 739     _BROADCASTS = {
 740         'archives': Broadcast('ARCHIVE', 'Past Broadcasts'),
 741         'highlights': Broadcast('HIGHLIGHT', 'Highlights'),
 742         'uploads': Broadcast('UPLOAD', 'Uploads'),
 743         'past_premieres': Broadcast('PAST_PREMIERE', 'Past Premieres'),
 744         'all': _DEFAULT_BROADCAST,
 745     }
 746
 747     _DEFAULT_SORTED_BY = 'Date'
 748     _SORTED_BY = {
 749         'time': _DEFAULT_SORTED_BY,
 750         'views': 'Popular',
 751     }
 752
 753     _OPERATION_NAME = 'FilterableVideoTower_Videos'
 754     _ENTRY_KIND = 'video'
 755     _EDGE_KIND = 'VideoEdge'
 756     _NODE_KIND = 'Video'
 757
 758     @classmethod
 759     def suitable(cls, url):
 760         return (False
 761                 if any(ie.suitable(url) for ie in (
 762                     TwitchVideosClipsIE,
 763                     TwitchVideosCollectionsIE))
 764                 else super().suitable(url))
 765
 766     @staticmethod
 767     def _make_variables(channel_name, broadcast_type, sort):
 768         return {
 769             'channelOwnerLogin': channel_name,
 770             'broadcastType': broadcast_type,
 771             'videoSort': sort.upper(),
 772         }
 773
 774     @staticmethod
 775     def _extract_entry(node):
 776         return _make_video_result(node)
 777
 778     def _real_extract(self, url):
 779         channel_name = self._match_id(url)
 780         qs = parse_qs(url)
 781         video_filter = qs.get('filter', ['all'])[0]
 782         sort = qs.get('sort', ['time'])[0]
 783         broadcast = self._BROADCASTS.get(video_filter, self._DEFAULT_BROADCAST)
 784         return self.playlist_result(
 785             self._entries(channel_name, broadcast.type, sort),
 786             playlist_id=channel_name,
 787             playlist_title=(
 788                 f'{channel_name} - {broadcast.label} '
 789                 f'sorted by {self._SORTED_BY.get(sort, self._DEFAULT_SORTED_BY)}'))
 790
 791
 792 class TwitchVideosClipsIE(TwitchPlaylistBaseIE):
 793     _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:clips|videos/*?\?.*?\bfilter=clips)'
 794
 795     _TESTS = [{
 796         # Clips
 797         'url': 'https://www.twitch.tv/vanillatv/clips?filter=clips&range=all',
 798         'info_dict': {
 799             'id': 'vanillatv',
 800             'title': 'vanillatv - Clips Top All',
 801         },
 802         'playlist_mincount': 1,
 803     }, {
 804         'url': 'https://www.twitch.tv/dota2ruhub/videos?filter=clips&range=7d',
 805         'only_matching': True,
 806     }]
 807
 808     Clip = collections.namedtuple('Clip', ['filter', 'label'])
 809
 810     _DEFAULT_CLIP = Clip('LAST_WEEK', 'Top 7D')
 811     _RANGE = {
 812         '24hr': Clip('LAST_DAY', 'Top 24H'),
 813         '7d': _DEFAULT_CLIP,
 814         '30d': Clip('LAST_MONTH', 'Top 30D'),
 815         'all': Clip('ALL_TIME', 'Top All'),
 816     }
 817
 818     # NB: values other than 20 result in skipped videos
 819     _PAGE_LIMIT = 20
 820
 821     _OPERATION_NAME = 'ClipsCards__User'
 822     _ENTRY_KIND = 'clip'
 823     _EDGE_KIND = 'ClipEdge'
 824     _NODE_KIND = 'Clip'
 825
 826     @staticmethod
 827     def _make_variables(channel_name, channel_filter):
 828         return {
 829             'login': channel_name,
 830             'criteria': {
 831                 'filter': channel_filter,
 832             },
 833         }
 834
 835     @staticmethod
 836     def _extract_entry(node):
 837         assert isinstance(node, dict)
 838         clip_url = url_or_none(node.get('url'))
 839         if not clip_url:
 840             return
 841         return {
 842             '_type': 'url_transparent',
 843             'ie_key': TwitchClipsIE.ie_key(),
 844             'id': node.get('id'),
 845             'url': clip_url,
 846             'title': node.get('title'),
 847             'thumbnail': node.get('thumbnailURL'),
 848             'duration': float_or_none(node.get('durationSeconds')),
 849             'timestamp': unified_timestamp(node.get('createdAt')),
 850             'view_count': int_or_none(node.get('viewCount')),
 851             'language': node.get('language'),
 852         }
 853
 854     def _real_extract(self, url):
 855         channel_name = self._match_id(url)
 856         qs = parse_qs(url)
 857         date_range = qs.get('range', ['7d'])[0]
 858         clip = self._RANGE.get(date_range, self._DEFAULT_CLIP)
 859         return self.playlist_result(
 860             self._entries(channel_name, clip.filter),
 861             playlist_id=channel_name,
 862             playlist_title=f'{channel_name} - Clips {clip.label}')
 863
 864
 865 class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
 866     _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/videos/*?\?.*?\bfilter=collections'
 867
 868     _TESTS = [{
 869         # Collections
 870         'url': 'https://www.twitch.tv/spamfish/videos?filter=collections',
 871         'info_dict': {
 872             'id': 'spamfish',
 873             'title': 'spamfish - Collections',
 874         },
 875         'playlist_mincount': 3,
 876     }, {
 877         'url': 'https://www.twitch.tv/monstercat/videos?filter=collections',
 878         'info_dict': {
 879             'id': 'monstercat',
 880             'title': 'monstercat - Collections',
 881         },
 882         'playlist_mincount': 13,
 883     }]
 884
 885     _OPERATION_NAME = 'ChannelCollectionsContent'
 886     _ENTRY_KIND = 'collection'
 887     _EDGE_KIND = 'CollectionsItemEdge'
 888     _NODE_KIND = 'Collection'
 889
 890     @staticmethod
 891     def _make_variables(channel_name):
 892         return {
 893             'ownerLogin': channel_name,
 894         }
 895
 896     @staticmethod
 897     def _extract_entry(node):
 898         assert isinstance(node, dict)
 899         collection_id = node.get('id')
 900         if not collection_id:
 901             return
 902         return {
 903             '_type': 'url_transparent',
 904             'ie_key': TwitchCollectionIE.ie_key(),
 905             'id': collection_id,
 906             'url': f'https://www.twitch.tv/collections/{collection_id}',
 907             'title': node.get('title'),
 908             'thumbnail': node.get('thumbnailURL'),
 909             'duration': float_or_none(node.get('lengthSeconds')),
 910             'timestamp': unified_timestamp(node.get('updatedAt')),
 911             'view_count': int_or_none(node.get('viewCount')),
 912         }
 913
 914     def _real_extract(self, url):
 915         channel_name = self._match_id(url)
 916         return self.playlist_result(
 917             self._entries(channel_name), playlist_id=channel_name,
 918             playlist_title=f'{channel_name} - Collections')
 919
 920
 921 class TwitchStreamIE(TwitchBaseIE):
 922     IE_NAME = 'twitch:stream'
 923     _VALID_URL = r'''(?x)
 924                     https?://
 925                         (?:
 926                             (?:(?:www|go|m)\.)?twitch\.tv/|
 927                             player\.twitch\.tv/\?.*?\bchannel=
 928                         )
 929                         (?P<id>[^/#?]+)
 930                     '''
 931
 932     _TESTS = [{
 933         'url': 'http://www.twitch.tv/shroomztv',
 934         'info_dict': {
 935             'id': '12772022048',
 936             'display_id': 'shroomztv',
 937             'ext': 'mp4',
 938             'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
 939             'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
 940             'is_live': True,
 941             'timestamp': 1421928037,
 942             'upload_date': '20150122',
 943             'uploader': 'ShroomzTV',
 944             'uploader_id': 'shroomztv',
 945             'view_count': int,
 946         },
 947         'params': {
 948             # m3u8 download
 949             'skip_download': True,
 950         },
 951         'skip': 'User does not exist',
 952     }, {
 953         'url': 'http://www.twitch.tv/miracle_doto#profile-0',
 954         'only_matching': True,
 955     }, {
 956         'url': 'https://player.twitch.tv/?channel=lotsofs',
 957         'only_matching': True,
 958     }, {
 959         'url': 'https://go.twitch.tv/food',
 960         'only_matching': True,
 961     }, {
 962         'url': 'https://m.twitch.tv/food',
 963         'only_matching': True,
 964     }, {
 965         'url': 'https://www.twitch.tv/monstercat',
 966         'info_dict': {
 967             'id': '40500071752',
 968             'display_id': 'monstercat',
 969             'title': 're:Monstercat',
 970             'description': 'md5:0945ad625e615bc8f0469396537d87d9',
 971             'is_live': True,
 972             'timestamp': 1677107190,
 973             'upload_date': '20230222',
 974             'uploader': 'Monstercat',
 975             'uploader_id': 'monstercat',
 976             'live_status': 'is_live',
 977             'thumbnail': 're:https://.*.jpg',
 978             'ext': 'mp4',
 979         },
 980         'params': {
 981             'skip_download': 'Livestream',
 982         },
 983     }]
 984
 985     @classmethod
 986     def suitable(cls, url):
 987         return (False
 988                 if any(ie.suitable(url) for ie in (
 989                     TwitchVodIE,
 990                     TwitchCollectionIE,
 991                     TwitchVideosIE,
 992                     TwitchVideosClipsIE,
 993                     TwitchVideosCollectionsIE,
 994                     TwitchClipsIE))
 995                 else super().suitable(url))
 996
 997     def _real_extract(self, url):
 998         channel_name = self._match_id(url).lower()
 999
1000         gql = self._download_gql(
1001             channel_name, [{
1002                 'operationName': 'StreamMetadata',
1003                 'variables': {'channelLogin': channel_name},
1004             }, {
1005                 'operationName': 'ComscoreStreamingQuery',
1006                 'variables': {
1007                     'channel': channel_name,
1008                     'clipSlug': '',
1009                     'isClip': False,
1010                     'isLive': True,
1011                     'isVodOrCollection': False,
1012                     'vodID': '',
1013                 },
1014             }, {
1015                 'operationName': 'VideoPreviewOverlay',
1016                 'variables': {'login': channel_name},
1017             }],
1018             'Downloading stream GraphQL')
1019
1020         user = gql[0]['data']['user']
1021
1022         if not user:
1023             raise ExtractorError(
1024                 f'{channel_name} does not exist', expected=True)
1025
1026         stream = user['stream']
1027
1028         if not stream:
1029             raise UserNotLive(video_id=channel_name)
1030
1031         access_token = self._download_access_token(
1032             channel_name, 'stream', 'channelName')
1033
1034         stream_id = stream.get('id') or channel_name
1035         formats = self._extract_twitch_m3u8_formats(
1036             'api/channel/hls', channel_name, access_token['value'], access_token['signature'])
1037         self._prefer_source(formats)
1038
1039         view_count = stream.get('viewers')
1040         timestamp = unified_timestamp(stream.get('createdAt'))
1041
1042         sq_user = try_get(gql, lambda x: x[1]['data']['user'], dict) or {}
1043         uploader = sq_user.get('displayName')
1044         description = try_get(
1045             sq_user, lambda x: x['broadcastSettings']['title'], str)
1046
1047         thumbnail = url_or_none(try_get(
1048             gql, lambda x: x[2]['data']['user']['stream']['previewImageURL'],
1049             str))
1050
1051         title = uploader or channel_name
1052         stream_type = stream.get('type')
1053         if stream_type in ['rerun', 'live']:
1054             title += f' ({stream_type})'
1055
1056         return {
1057             'id': stream_id,
1058             'display_id': channel_name,
1059             'title': title,
1060             'description': description,
1061             'thumbnails': self._get_thumbnails(thumbnail),
1062             'uploader': uploader,
1063             'uploader_id': channel_name,
1064             'timestamp': timestamp,
1065             'view_count': view_count,
1066             'formats': formats,
1067             'is_live': stream_type == 'live',
1068         }
1069
1070
1071 class TwitchClipsIE(TwitchBaseIE):
1072     IE_NAME = 'twitch:clips'
1073     _VALID_URL = r'''(?x)
1074                     https?://
1075                         (?:
1076                             clips\.twitch\.tv/(?:embed\?.*?\bclip=|(?:[^/]+/)*)|
1077                             (?:(?:www|go|m)\.)?twitch\.tv/(?:[^/]+/)?clip/
1078                         )
1079                         (?P<id>[^/?#&]+)
1080                     '''
1081
1082     _TESTS = [{
1083         'url': 'https://clips.twitch.tv/FaintLightGullWholeWheat',
1084         'md5': '761769e1eafce0ffebfb4089cb3847cd',
1085         'info_dict': {
1086             'id': '42850523',
1087             'display_id': 'FaintLightGullWholeWheat',
1088             'ext': 'mp4',
1089             'title': 'EA Play 2016 Live from the Novo Theatre',
1090             'thumbnail': r're:^https?://.*\.jpg',
1091             'timestamp': 1465767393,
1092             'upload_date': '20160612',
1093             'creator': 'EA',
1094             'uploader': 'stereotype_',
1095             'uploader_id': '43566419',
1096         },
1097     }, {
1098         # multiple formats
1099         'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
1100         'only_matching': True,
1101     }, {
1102         'url': 'https://www.twitch.tv/sergeynixon/clip/StormyThankfulSproutFutureMan',
1103         'only_matching': True,
1104     }, {
1105         'url': 'https://clips.twitch.tv/embed?clip=InquisitiveBreakableYogurtJebaited',
1106         'only_matching': True,
1107     }, {
1108         'url': 'https://m.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
1109         'only_matching': True,
1110     }, {
1111         'url': 'https://go.twitch.tv/rossbroadcast/clip/ConfidentBraveHumanChefFrank',
1112         'only_matching': True,
1113     }, {
1114         'url': 'https://m.twitch.tv/clip/FaintLightGullWholeWheat',
1115         'only_matching': True,
1116     }]
1117
1118     def _real_extract(self, url):
1119         video_id = self._match_id(url)
1120
1121         clip = self._download_gql(
1122             video_id, [{
1123                 'operationName': 'VideoAccessToken_Clip',
1124                 'variables': {
1125                     'slug': video_id,
1126                 },
1127             }],
1128             'Downloading clip access token GraphQL')[0]['data']['clip']
1129
1130         if not clip:
1131             raise ExtractorError(
1132                 'This clip is no longer available', expected=True)
1133
1134         access_query = {
1135             'sig': clip['playbackAccessToken']['signature'],
1136             'token': clip['playbackAccessToken']['value'],
1137         }
1138
1139         data = self._download_base_gql(
1140             video_id, {
1141                 'query': '''{
1142   clip(slug: "%s") {
1143     broadcaster {
1144       displayName
1145     }
1146     createdAt
1147     curator {
1148       displayName
1149       id
1150     }
1151     durationSeconds
1152     id
1153     tiny: thumbnailURL(width: 86, height: 45)
1154     small: thumbnailURL(width: 260, height: 147)
1155     medium: thumbnailURL(width: 480, height: 272)
1156     title
1157     videoQualities {
1158       frameRate
1159       quality
1160       sourceURL
1161     }
1162     viewCount
1163   }
1164 }''' % video_id}, 'Downloading clip GraphQL', fatal=False)  # noqa: UP031
1165
1166         if data:
1167             clip = try_get(data, lambda x: x['data']['clip'], dict) or clip
1168
1169         formats = []
1170         for option in clip.get('videoQualities', []):
1171             if not isinstance(option, dict):
1172                 continue
1173             source = url_or_none(option.get('sourceURL'))
1174             if not source:
1175                 continue
1176             formats.append({
1177                 'url': update_url_query(source, access_query),
1178                 'format_id': option.get('quality'),
1179                 'height': int_or_none(option.get('quality')),
1180                 'fps': int_or_none(option.get('frameRate')),
1181             })
1182
1183         thumbnails = []
1184         for thumbnail_id in ('tiny', 'small', 'medium'):
1185             thumbnail_url = clip.get(thumbnail_id)
1186             if not thumbnail_url:
1187                 continue
1188             thumb = {
1189                 'id': thumbnail_id,
1190                 'url': thumbnail_url,
1191             }
1192             mobj = re.search(r'-(\d+)x(\d+)\.', thumbnail_url)
1193             if mobj:
1194                 thumb.update({
1195                     'height': int(mobj.group(2)),
1196                     'width': int(mobj.group(1)),
1197                 })
1198             thumbnails.append(thumb)
1199
1200         old_id = self._search_regex(r'%7C(\d+)(?:-\d+)?.mp4', formats[-1]['url'], 'old id', default=None)
1201
1202         return {
1203             'id': clip.get('id') or video_id,
1204             '_old_archive_ids': [make_archive_id(self, old_id)] if old_id else None,
1205             'display_id': video_id,
1206             'title': clip.get('title'),
1207             'formats': formats,
1208             'duration': int_or_none(clip.get('durationSeconds')),
1209             'view_count': int_or_none(clip.get('viewCount')),
1210             'timestamp': unified_timestamp(clip.get('createdAt')),
1211             'thumbnails': thumbnails,
1212             'creator': try_get(clip, lambda x: x['broadcaster']['displayName'], str),
1213             'uploader': try_get(clip, lambda x: x['curator']['displayName'], str),
1214             'uploader_id': try_get(clip, lambda x: x['curator']['id'], str),
1215         }