]> jfr.im git - yt-dlp.git/blobdiff - yt_dlp/extractor/yahoo.py
[generic] Extract m3u8 formats from JSON-LD
[yt-dlp.git] / yt_dlp / extractor / yahoo.py
index ecf2f5f48bdea3b5949cbc5ddc46260109a03e22..313e596f56e6ff432af40ba8f4c0cc72d9e1b566 100644 (file)
@@ -22,6 +22,7 @@
 )
 
 from .brightcove import BrightcoveNewIE
+from .youtube import YoutubeIE
 
 
 class YahooIE(InfoExtractor):
@@ -38,6 +39,7 @@ class YahooIE(InfoExtractor):
             'timestamp': 1369812016,
             'upload_date': '20130529',
         },
+        'skip': 'No longer exists',
     }, {
         'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
         'md5': '7993e572fac98e044588d0b5260f4352',
@@ -50,6 +52,7 @@ class YahooIE(InfoExtractor):
             'timestamp': 1406838636,
             'upload_date': '20140731',
         },
+        'skip': 'Unfortunately, this video is not available in your region',
     }, {
         'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
         'md5': '71298482f7c64cbb7fa064e4553ff1c1',
@@ -61,7 +64,8 @@ class YahooIE(InfoExtractor):
             'duration': 97,
             'timestamp': 1414489862,
             'upload_date': '20141028',
-        }
+        },
+        'skip': 'No longer exists',
     }, {
         'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
         'md5': '88e209b417f173d86186bef6e4d1f160',
@@ -120,6 +124,7 @@ class YahooIE(InfoExtractor):
             'season_number': 6,
             'episode_number': 1,
         },
+        'skip': 'No longer exists',
     }, {
         # ytwnews://cavideo/
         'url': 'https://tw.video.yahoo.com/movie-tw/單車天使-中文版預-092316541.html',
@@ -156,7 +161,7 @@ class YahooIE(InfoExtractor):
                 'id': '352CFDOQrKg',
                 'ext': 'mp4',
                 'title': 'Kyndal Inskeep "Performs the Hell Out of" Sia\'s "Elastic Heart" - The Voice Knockouts 2019',
-                'description': 'md5:35b61e94c2ae214bc965ff4245f80d11',
+                'description': 'md5:7fe8e3d5806f96002e55f190d1d94479',
                 'uploader': 'The Voice',
                 'uploader_id': 'NBCTheVoice',
                 'upload_date': '20191029',
@@ -165,7 +170,7 @@ class YahooIE(InfoExtractor):
         'params': {
             'playlistend': 2,
         },
-        'expected_warnings': ['HTTP Error 404'],
+        'expected_warnings': ['HTTP Error 404', 'Ignoring subtitle tracks'],
     }, {
         'url': 'https://malaysia.news.yahoo.com/video/bystanders-help-ontario-policeman-bust-190932818.html',
         'only_matching': True,
@@ -259,7 +264,7 @@ def _extract_yahoo_video(self, video_id, country):
 
         return {
             'id': video_id,
-            'title': self._live_title(title) if is_live else title,
+            'title': title,
             'formats': formats,
             'thumbnails': thumbnails,
             'description': clean_html(video.get('description')),
@@ -274,18 +279,19 @@ def _extract_yahoo_video(self, video_id, country):
         }
 
     def _real_extract(self, url):
-        url, country, display_id = re.match(self._VALID_URL, url).groups()
+        url, country, display_id = self._match_valid_url(url).groups()
         if not country:
             country = 'us'
         else:
             country = country.split('-')[0]
 
-        item = self._download_json(
+        items = self._download_json(
             'https://%s.yahoo.com/caas/content/article' % country, display_id,
             'Downloading content JSON metadata', query={
                 'url': url
-            })['items'][0]['data']['partnerData']
+            })['items'][0]
 
+        item = items['data']['partnerData']
         if item.get('type') != 'video':
             entries = []
 
@@ -299,9 +305,19 @@ def _real_extract(self, url):
             for e in (item.get('body') or []):
                 if e.get('type') == 'videoIframe':
                     iframe_url = e.get('url')
-                    if not iframe_url:
-                        continue
+                    if iframe_url:
+                        entries.append(self.url_result(iframe_url))
+
+            if item.get('type') == 'storywithleadvideo':
+                iframe_url = try_get(item, lambda x: x['meta']['player']['url'])
+                if iframe_url:
                     entries.append(self.url_result(iframe_url))
+                else:
+                    self.report_warning("Yahoo didn't provide an iframe url for this storywithleadvideo")
+
+            if items.get('markup'):
+                entries.extend(
+                    self.url_result(yt_url) for yt_url in YoutubeIE._extract_urls(items['markup']))
 
             return self.playlist_result(
                 entries, item.get('uuid'),
@@ -318,35 +334,19 @@ class YahooSearchIE(SearchInfoExtractor):
     IE_NAME = 'screen.yahoo:search'
     _SEARCH_KEY = 'yvsearch'
 
-    def _get_n_results(self, query, n):
-        """Get a specified number of results for a query"""
-        entries = []
+    def _search_results(self, query):
         for pagenum in itertools.count(0):
             result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
             info = self._download_json(result_url, query,
                                        note='Downloading results page ' + str(pagenum + 1))
-            m = info['m']
-            results = info['results']
-
-            for (i, r) in enumerate(results):
-                if (pagenum * 30) + i >= n:
-                    break
-                mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
-                e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
-                entries.append(e)
-            if (pagenum * 30 + i >= n) or (m['last'] >= (m['total'] - 1)):
+            yield from (self.url_result(result['rurl']) for result in info['results'])
+            if info['m']['last'] >= info['m']['total'] - 1:
                 break
 
-        return {
-            '_type': 'playlist',
-            'id': query,
-            'entries': entries,
-        }
-
 
 class YahooGyaOPlayerIE(InfoExtractor):
     IE_NAME = 'yahoo:gyao:player'
-    _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:player|episode/[^/]+)|streaming\.yahoo\.co\.jp/c/y)/(?P<id>\d+/v\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:player|episode(?:/[^/]+)?)|streaming\.yahoo\.co\.jp/c/y)/(?P<id>\d+/v\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
     _TESTS = [{
         'url': 'https://gyao.yahoo.co.jp/player/00998/v00818/v0000000000000008564/',
         'info_dict': {
@@ -368,6 +368,9 @@ class YahooGyaOPlayerIE(InfoExtractor):
     }, {
         'url': 'https://gyao.yahoo.co.jp/episode/%E3%81%8D%E3%81%AE%E3%81%86%E4%BD%95%E9%A3%9F%E3%81%B9%E3%81%9F%EF%BC%9F%20%E7%AC%AC2%E8%A9%B1%202019%2F4%2F12%E6%94%BE%E9%80%81%E5%88%86/5cb02352-b725-409e-9f8d-88f947a9f682',
         'only_matching': True,
+    }, {
+        'url': 'https://gyao.yahoo.co.jp/episode/5fa1226c-ef8d-4e93-af7a-fd92f4e30597',
+        'only_matching': True,
     }]
     _GEO_BYPASS = False
 
@@ -508,7 +511,7 @@ def _extract_formats(self, json_data, content_id):
         return formats
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        mobj = self._match_valid_url(url)
         host = mobj.group('host')
         display_id = mobj.group('id') or host