[generic] Extract m3u8 formats from JSON-LD

[yt-dlp.git] / yt_dlp / extractor / twitch.py

diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index e544e47371faf9261bca877226423925ec4af241..54e500edd03ba37d8b29b4266d12c1066a32cb95 100644 (file)
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -11,7 +11,6 @@
 from ..compat import (
     compat_parse_qs,
     compat_str,
-    compat_urlparse,
     compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
 )
@@ -23,7 +22,10 @@
     int_or_none,
     parse_duration,
     parse_iso8601,
+    parse_qs,
     qualities,
+    str_or_none,
+    traverse_obj,
     try_get,
     unified_timestamp,
     update_url_query,
@@ -52,6 +54,7 @@ class TwitchBaseIE(InfoExtractor):
         'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
         'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
         'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
+        'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
     }
 
     def _real_initialize(self):
@@ -249,6 +252,38 @@ class TwitchVodIE(TwitchBaseIE):
     }, {
         'url': 'https://player.twitch.tv/?video=480452374',
         'only_matching': True,
+    }, {
+        'url': 'https://www.twitch.tv/videos/635475444',
+        'info_dict': {
+            'id': 'v635475444',
+            'ext': 'mp4',
+            'title': 'Riot Games',
+            'duration': 11643,
+            'uploader': 'Riot Games',
+            'uploader_id': 'riotgames',
+            'timestamp': 1590770569,
+            'upload_date': '20200529',
+            'chapters': [
+                {
+                    'start_time': 0,
+                    'end_time': 573,
+                    'title': 'League of Legends'
+                },
+                {
+                    'start_time': 573,
+                    'end_time': 3922,
+                    'title': 'Legends of Runeterra'
+                },
+                {
+                    'start_time': 3922,
+                    'end_time': 11643,
+                    'title': 'Art'
+                }
+            ],
+        },
+        'params': {
+            'skip_download': True
+        }
     }]
 
     def _download_info(self, item_id):
@@ -259,16 +294,24 @@ def _download_info(self, item_id):
                     'channelLogin': '',
                     'videoID': item_id,
                 },
+            }, {
+                'operationName': 'VideoPlayer_ChapterSelectButtonVideo',
+                'variables': {
+                    'includePrivate': False,
+                    'videoID': item_id,
+                },
             }],
-            'Downloading stream metadata GraphQL')[0]['data']
-        video = data.get('video')
+            'Downloading stream metadata GraphQL')
+
+        video = traverse_obj(data, (0, 'data', 'video'))
+        video['moments'] = traverse_obj(data, (1, 'data', 'video', 'moments', 'edges', ..., 'node'))
+
         if video is None:
             raise ExtractorError(
                 'Video %s does not exist' % item_id, expected=True)
         return self._extract_info_gql(video, item_id)
 
-    @staticmethod
-    def _extract_info(info):
+    def _extract_info(self, info):
         status = info.get('status')
         if status == 'recording':
             is_live = True
@@ -302,18 +345,39 @@ def _extract_info(info):
             'timestamp': parse_iso8601(info.get('recorded_at')),
             'view_count': int_or_none(info.get('views')),
             'is_live': is_live,
+            'was_live': True,
         }
 
-    @staticmethod
-    def _extract_info_gql(info, item_id):
+    def _extract_moments(self, info, item_id):
+        for moment in info.get('moments') or []:
+            start_time = int_or_none(moment.get('positionMilliseconds'), 1000)
+            duration = int_or_none(moment.get('durationMilliseconds'), 1000)
+            name = str_or_none(moment.get('description'))
+
+            if start_time is None or duration is None:
+                self.report_warning(f'Important chapter information missing for chapter {name}', item_id)
+                continue
+            yield {
+                'start_time': start_time,
+                'end_time': start_time + duration,
+                'title': name,
+            }
+
+    def _extract_info_gql(self, info, item_id):
         vod_id = info.get('id') or item_id
         # id backward compatibility for download archives
         if vod_id[0] != 'v':
             vod_id = 'v%s' % vod_id
         thumbnail = url_or_none(info.get('previewThumbnailURL'))
+        is_live = None
         if thumbnail:
-            for p in ('width', 'height'):
-                thumbnail = thumbnail.replace('{%s}' % p, '0')
+            if thumbnail.endswith('/404_processing_{width}x{height}.png'):
+                is_live, thumbnail = True, None
+            else:
+                is_live = False
+                for p in ('width', 'height'):
+                    thumbnail = thumbnail.replace('{%s}' % p, '0')
+
         return {
             'id': vod_id,
             'title': info.get('title') or 'Untitled Broadcast',
@@ -324,6 +388,9 @@ def _extract_info_gql(info, item_id):
             'uploader_id': try_get(info, lambda x: x['owner']['login'], compat_str),
             'timestamp': unified_timestamp(info.get('publishedAt')),
             'view_count': int_or_none(info.get('viewCount')),
+            'chapters': list(self._extract_moments(info, item_id)),
+            'is_live': is_live,
+            'was_live': True,
         }
 
     def _real_extract(self, url):
@@ -571,7 +638,7 @@ def _extract_entry(node):
 
     def _real_extract(self, url):
         channel_name = self._match_id(url)
-        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        qs = parse_qs(url)
         filter = qs.get('filter', ['all'])[0]
         sort = qs.get('sort', ['time'])[0]
         broadcast = self._BROADCASTS.get(filter, self._DEFAULT_BROADCAST)
@@ -647,7 +714,7 @@ def _extract_entry(node):
 
     def _real_extract(self, url):
         channel_name = self._match_id(url)
-        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        qs = parse_qs(url)
         range = qs.get('range', ['7d'])[0]
         clip = self._RANGE.get(range, self._DEFAULT_CLIP)
         return self.playlist_result(
@@ -836,7 +903,7 @@ def _real_extract(self, url):
         return {
             'id': stream_id,
             'display_id': channel_name,
-            'title': self._live_title(title),
+            'title': title,
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,