5 from .brightcove
import BrightcoveNewIE
6 from .common
import InfoExtractor
, SearchInfoExtractor
7 from .youtube
import YoutubeIE
21 class YahooIE(InfoExtractor
):
22 IE_DESC
= 'Yahoo screen and movies'
23 _VALID_URL
= r
'(?P<url>https?://(?:(?P<country>[a-zA-Z]{2}(?:-[a-zA-Z]{2})?|malaysia)\.)?(?:[\da-zA-Z_-]+\.)?yahoo\.com/(?:[^/]+/)*(?P<id>[^?&#]*-[0-9]+(?:-[a-z]+)?)\.html)'
24 _EMBED_REGEX
= [r
'<iframe[^>]+?src=(["\'])(?P
<url
>https?
://(?
:screen|movies
)\
.yahoo\
.com
/.+?\
.html
\?format
=embed
)\
1']
27 'url
': 'http
://screen
.yahoo
.com
/julian
-smith
-travis
-legg
-watch
-214727115.html
',
29 'id': '2d25e626
-2378-391f
-ada0
-ddaf1417e588
',
31 'title
': 'Julian Smith
& Travis Legg Watch Julian Smith
',
32 'description
': 'Julian
and Travis watch Julian Smith
',
34 'timestamp
': 1369812016,
35 'upload_date
': '20130529',
37 'skip
': 'No longer exists
',
39 'url
': 'https
://screen
.yahoo
.com
/community
/community
-sizzle
-reel
-203225340.html?format
=embed
',
40 'md5
': '7993e572fac98e044588d0b5260f4352
',
42 'id': '4fe78544
-8d48
-39d8
-97cd
-13f205d9fcdb
',
44 'title
': "Yahoo Saves 'Community
'",
45 'description
': 'md5
:4d4145af2fd3de00cbb6c1d664105053
',
47 'timestamp
': 1406838636,
48 'upload_date
': '20140731',
50 'skip
': 'Unfortunately
, this video
is not available
in your region
',
52 'url
': 'https
://uk
.screen
.yahoo
.com
/editor
-picks
/cute
-raccoon
-freed
-drain
-using
-091756545.html
',
53 'md5
': '71298482f7c64cbb7fa064e4553ff1c1
',
55 'id': 'b3affa53
-2e14
-3590-852b
-0e0db6cd1a58
',
57 'title
': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder
',
58 'description
': 'md5
:f66c890e1490f4910a9953c941dee944
',
60 'timestamp
': 1414489862,
61 'upload_date
': '20141028',
63 'skip
': 'No longer exists
',
65 'url
': 'http
://news
.yahoo
.com
/video
/china
-moses
-crazy
-blues
-104538833.html
',
66 'md5
': '88e209b417f173d86186bef6e4d1f160
',
68 'id': 'f885cf7f
-43d4
-3450-9fac
-46ac30ece521
',
70 'title
': 'China Moses Is Crazy About the Blues
',
71 'description
': 'md5
:9900ab8cd5808175c7b3fe55b979bed0
',
73 'timestamp
': 1385722202,
74 'upload_date
': '20131129',
77 'url
': 'https
://www
.yahoo
.com
/movies
/v
/true
-story
-trailer
-173000497.html
',
78 'md5
': '2a9752f74cb898af5d1083ea9f661b58
',
80 'id': '071c4013
-ce30
-3a93
-a5b2
-e0413cd4a9d1
',
82 'title
': '\'True Story
\' Trailer
',
83 'description
': 'True Story
',
85 'timestamp
': 1418919206,
86 'upload_date
': '20141218',
89 'url
': 'https
://gma
.yahoo
.com
/pizza
-delivery
-man
-surprised
-huge
-tip
-college
-kids
-195200785.html
',
90 'only_matching
': True,
92 'note
': 'NBC Sports embeds
',
93 'url
': 'http
://sports
.yahoo
.com
/blogs
/ncaab
-the
-dagger
/tyler
-kalinoski
-s
-buzzer
-beater
-caps
-davidson
-s
-comeback
-win
-185609842.html?guid
=nbc_cbk_davidsonbuzzerbeater_150313
',
97 'description
': 'md5
:df390f70a9ba7c95ff1daace988f0d8d
',
98 'title
': 'Tyler Kalinoski hits buzzer
-beater to lift Davidson
',
99 'upload_date
': '20150313',
100 'uploader
': 'NBCU
-SPORTS
',
101 'timestamp
': 1426270238,
104 'url
': 'https
://tw
.news
.yahoo
.com
/-100120367.html
',
105 'only_matching
': True,
107 # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
108 'url
': 'https
://screen
.yahoo
.com
/community
/communitary
-community
-episode
-1-ladders
-154501237.html
',
109 'md5
': '4fbafb9c9b6f07aa8f870629f6671b35
',
111 'id': '1f32853c
-a271
-3eef
-8cb6
-f6d6872cb504
',
113 'title
': 'Communitary
- Community Episode
1: Ladders
',
114 'description
': 'md5
:8fc39608213295748e1e289807838c97
',
116 'timestamp
': 1440436550,
117 'upload_date
': '20150824',
118 'series
': 'Communitary
',
122 'skip
': 'No longer exists
',
125 'url
': 'https
://tw
.video
.yahoo
.com
/movie
-tw
/單車天使
-中文版預
-092316541.html
',
127 'id': 'ba133ff2
-0793-3510-b636
-59dfe9ff6cff
',
129 'title
': '單車天使
- 中文版預
',
130 'description
': '中文版預
',
131 'timestamp
': 1476696196,
132 'upload_date
': '20161017',
135 'skip_download
': True,
138 # Contains both a Yahoo hosted video and multiple Youtube embeds
139 'url
': 'https
://www
.yahoo
.com
/entertainment
/gwen
-stefani
-reveals
-the
-pop
-hit
-she
-passed
-on
-assigns
-it
-to
-her
-voice
-contestant
-instead
-033045672.html
',
141 'id': '46c5d95a
-528f
-3d03
-b732
-732fcadd51de
',
142 'title
': 'Gwen Stefani reveals the pop hit she passed on
, assigns it to her
\'Voice
\' contestant instead
',
143 'description
': 'Gwen decided
not to record this hit herself
, but she decided it was the perfect fit
for Kyndall Inskeep
.',
147 'id': '966d4262
-4fd1
-3aaa
-b45b
-049ca6e38ba6
',
149 'title
': 'Gwen Stefani reveals she turned down one of Sia
\'s best songs
',
150 'description
': 'On
"The Voice" Tuesday
, Gwen Stefani told Taylor Swift which Sia hit was almost hers
.',
151 'timestamp
': 1572406500,
152 'upload_date
': '20191030',
158 'title
': 'Kyndal Inskeep
"Performs the Hell Out of" Sia
\'s
"Elastic Heart" - The Voice Knockouts
2019',
159 'description
': 'md5
:7fe8e3d5806f96002e55f190d1d94479
',
160 'uploader
': 'The Voice
',
161 'uploader_id
': 'NBCTheVoice
',
162 'upload_date
': '20191029',
168 'expected_warnings
': ['HTTP Error
404', 'Ignoring subtitle tracks
'],
170 'url
': 'https
://malaysia
.news
.yahoo
.com
/video
/bystanders
-help-ontario
-policeman
-bust
-190932818.html
',
171 'only_matching
': True,
173 'url
': 'https
://es
-us
.noticias
.yahoo
.com
/es
-la
-puerta
-irrompible
-que
-110539379.html
',
174 'only_matching
': True,
176 'url
': 'https
://www
.yahoo
.com
/entertainment
/v
/longtime
-cbs
-news
-60-minutes
-032036500-cbs
.html
',
177 'only_matching
': True,
180 def _extract_yahoo_video(self, video_id, country):
181 video = self._download_json(
182 'https
://%s.yahoo
.com
/_td
/api
/resource
/VideoService
.videos
;view
=full
;video_ids
=["%s"]' % (country, video_id),
183 video_id, 'Downloading video JSON metadata
')[0]
184 title = video['title
']
186 if country == 'malaysia
':
189 is_live = video.get('live_state
') == 'live
'
190 fmts = ('m3u8
',) if is_live else ('webm
', 'mp4
')
196 media_obj = self._download_json(
197 'https
://video
-api
.yql
.yahoo
.com
/v1
/video
/sapi
/streams
/' + video_id,
198 video_id, 'Downloading
%s JSON metadata
' % fmt,
199 headers=self.geo_verification_headers(), query={
201 'region
': country.upper(),
202 })['query
']['results
']['mediaObj
'][0]
203 msg = media_obj.get('status
', {}).get('msg
')
205 for s in media_obj.get('streams
', []):
208 if not host or not path:
211 if s.get('format
') == 'm3u8
':
212 formats.extend(self._extract_m3u8_formats(
213 s_url, video_id, 'mp4
', m3u8_id='hls
', fatal=False))
215 tbr = int_or_none(s.get('bitrate
'))
218 'format_id
': fmt + ('-%d' % tbr if tbr else ''),
219 'width
': int_or_none(s.get('width
')),
220 'height
': int_or_none(s.get('height
')),
222 'fps
': int_or_none(s.get('framerate
')),
225 for cc in media_obj.get('closedcaptions
', []):
226 cc_url = cc.get('url
')
227 if not cc_url or cc_url in urls:
230 subtitles.setdefault(cc.get('lang
') or 'en
-US
', []).append({
232 'ext
': mimetype2ext(cc.get('content_type
')),
235 streaming_url = video.get('streaming_url
')
236 if streaming_url and not is_live:
237 formats.extend(self._extract_m3u8_formats(
238 streaming_url, video_id, 'mp4
',
239 'm3u8_native
', m3u8_id='hls
', fatal=False))
241 if not formats and msg == 'geo restricted
':
242 self.raise_geo_restricted(metadata_available=True)
244 self._sort_formats(formats)
247 for thumb in video.get('thumbnails
', []):
248 thumb_url = thumb.get('url
')
252 'id': thumb.get('tag
'),
253 'url
': thumb.get('url
'),
254 'width
': int_or_none(thumb.get('width
')),
255 'height
': int_or_none(thumb.get('height
')),
258 series_info = video.get('series_info
') or {}
264 'thumbnails
': thumbnails,
265 'description
': clean_html(video.get('description
')),
266 'timestamp
': parse_iso8601(video.get('publish_time
')),
267 'subtitles
': subtitles,
268 'duration
': int_or_none(video.get('duration
')),
269 'view_count
': int_or_none(video.get('view_count
')),
271 'series
': video.get('show_name
'),
272 'season_number
': int_or_none(series_info.get('season_number
')),
273 'episode_number
': int_or_none(series_info.get('episode_number
')),
276 def _real_extract(self, url):
277 url, country, display_id = self._match_valid_url(url).groups()
281 country = country.split('-')[0]
283 items = self._download_json(
284 'https
://%s.yahoo
.com
/caas
/content
/article
' % country, display_id,
285 'Downloading content JSON metadata
', query={
289 item = items['data
']['partnerData
']
290 if item.get('type') != 'video
':
293 cover = item.get('cover
') or {}
294 if cover.get('type') == 'yvideo
':
295 cover_url = cover.get('url
')
297 entries.append(self.url_result(
298 cover_url, 'Yahoo
', cover.get('uuid
')))
300 for e in (item.get('body
') or []):
301 if e.get('type') == 'videoIframe
':
302 iframe_url = e.get('url
')
304 entries.append(self.url_result(iframe_url))
306 if item.get('type') == 'storywithleadvideo
':
307 iframe_url = try_get(item, lambda x: x['meta
']['player
']['url
'])
309 entries.append(self.url_result(iframe_url))
311 self.report_warning("Yahoo didn't provide an iframe url
for this storywithleadvideo
")
313 if items.get('markup'):
315 self.url_result(yt_url) for yt_url in YoutubeIE._extract_embed_urls(url, items['markup']))
317 return self.playlist_result(
318 entries, item.get('uuid'),
319 item.get('title'), item.get('summary'))
321 info = self._extract_yahoo_video(item['uuid'], country)
322 info['display_id'] = display_id
326 class YahooSearchIE(SearchInfoExtractor):
327 IE_DESC = 'Yahoo screen search'
329 IE_NAME = 'screen.yahoo:search'
330 _SEARCH_KEY = 'yvsearch'
332 def _search_results(self, query):
333 for pagenum in itertools.count(0):
334 result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (urllib.parse.quote_plus(query), pagenum * 30)
335 info = self._download_json(result_url, query,
336 note='Downloading results page ' + str(pagenum + 1))
337 yield from (self.url_result(result['rurl']) for result in info['results'])
338 if info['m']['last'] >= info['m']['total'] - 1:
342 class YahooGyaOPlayerIE(InfoExtractor):
343 IE_NAME = 'yahoo:gyao:player'
344 _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:player|episode(?:/[^/]+)?)|streaming\.yahoo\.co\.jp/c/y)/(?P<id>\d+/v\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
346 'url': 'https://gyao.yahoo.co.jp/player/00998/v00818/v0000000000000008564/',
348 'id': '5993125228001',
350 'title': 'フューリー 【字幕版】',
351 'description': 'md5:21e691c798a15330eda4db17a8fe45a5',
352 'uploader_id': '4235717419001',
353 'upload_date': '20190124',
354 'timestamp': 1548294365,
358 'skip_download': True,
361 'url': 'https://streaming.yahoo.co.jp/c/y/01034/v00133/v0000000000000000706/',
362 'only_matching': True,
364 'url': 'https://gyao.yahoo.co.jp/episode/%E3%81%8D%E3%81%AE%E3%81%86%E4%BD%95%E9%A3%9F%E3%81%B9%E3%81%9F%EF%BC%9F%20%E7%AC%AC2%E8%A9%B1%202019%2F4%2F12%E6%94%BE%E9%80%81%E5%88%86/5cb02352-b725-409e-9f8d-88f947a9f682',
365 'only_matching': True,
367 'url': 'https://gyao.yahoo.co.jp/episode/5fa1226c-ef8d-4e93-af7a-fd92f4e30597',
368 'only_matching': True,
372 def _real_extract(self, url):
373 video_id = self._match_id(url).replace('/', ':')
374 headers = self.geo_verification_headers()
375 headers['Accept'] = 'application/json'
376 resp = self._download_json(
377 'https://gyao.yahoo.co.jp/apis/playback/graphql', video_id, query={
378 'appId': 'dj00aiZpPUNJeDh2cU1RazU3UCZzPWNvbnN1bWVyc2VjcmV0Jng9NTk-',
380 content(parameter: {contentId: "%s", logicaAgent: PC_WEB}) {
390 content = resp['data']['content']
392 msg = resp['errors'][0]['message']
393 if msg == 'not in japan':
394 self.raise_geo_restricted(countries=['JP'])
395 raise ExtractorError(msg)
396 video = content['video']
398 '_type': 'url_transparent',
400 'title': video['title'],
402 'http://players.brightcove.net/4235717419001/SyG5P0gjb_default/index.html?videoId=' + video['delivery']['id'],
403 {'geo_countries': ['JP']}),
404 'ie_key': BrightcoveNewIE.ie_key(),
408 class YahooGyaOIE(InfoExtractor):
409 IE_NAME = 'yahoo:gyao'
410 _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:p|title(?:/[^/]+)?)|streaming\.yahoo\.co\.jp/p/y)/(?P<id>\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
412 'url': 'https://gyao.yahoo.co.jp/title/%E3%82%BF%E3%82%A4%E3%83%A0%E3%83%9C%E3%82%AB%E3%83%B3%E3%82%B7%E3%83%AA%E3%83%BC%E3%82%BA%20%E3%83%A4%E3%83%83%E3%82%BF%E3%83%BC%E3%83%9E%E3%83%B3/5f60ceb3-6e5e-40ef-ba40-d68b598d067f',
414 'id': '5f60ceb3-6e5e-40ef-ba40-d68b598d067f',
416 'playlist_mincount': 80,
418 'url': 'https://gyao.yahoo.co.jp/p/00449/v03102/',
419 'only_matching': True,
421 'url': 'https://streaming.yahoo.co.jp/p/y/01034/v00133/',
422 'only_matching': True,
424 'url': 'https://gyao.yahoo.co.jp/title/%E3%81%97%E3%82%83%E3%81%B9%E3%81%8F%E3%82%8A007/5b025a49-b2e5-4dc7-945c-09c6634afacf',
425 'only_matching': True,
427 'url': 'https://gyao.yahoo.co.jp/title/5b025a49-b2e5-4dc7-945c-09c6634afacf',
428 'only_matching': True,
431 def _entries(self, program_id):
434 playlist = self._download_json(
435 f'https://gyao.yahoo.co.jp/api/programs/{program_id}/videos?page={page}&serviceId=gy', program_id,
436 note=f'Downloading JSON metadata page {page}')
439 for video in playlist['videos']:
440 video_id = video.get('id')
443 if video.get('streamingAvailability') == 'notYet':
445 yield self.url_result(
446 'https://gyao.yahoo.co.jp/player/%s/' % video_id.replace(':', '/'),
447 YahooGyaOPlayerIE.ie_key(), video_id)
448 if playlist.get('ended'):
452 def _real_extract(self, url):
453 program_id = self._match_id(url).replace('/', ':')
454 return self.playlist_result(self._entries(program_id), program_id)
457 class YahooJapanNewsIE(InfoExtractor):
458 IE_NAME = 'yahoo:japannews'
459 IE_DESC = 'Yahoo! Japan News'
460 _VALID_URL = r'https?://news\.yahoo\.co\.jp/(?:articles|feature)/(?P<id>[a-zA-Z0-9]+)'
461 _GEO_COUNTRIES = ['JP']
463 'url': 'https://news.yahoo.co.jp/articles/a70fe3a064f1cfec937e2252c7fc6c1ba3201c0e',
465 'id': 'a70fe3a064f1cfec937e2252c7fc6c1ba3201c0e',
467 'title': '【独自】安倍元総理「国葬」中止求め“脅迫メール”…「子ども誘拐」“送信者”を追跡',
468 'description': 'md5:1c06974575f930f692d8696fbcfdc546',
469 'thumbnail': r're:https://.+',
472 'skip_download': True,
475 'url': 'https://news.yahoo.co.jp/feature/1356',
476 'only_matching': True
479 def _extract_formats(self, json_data, content_id):
482 for vid in traverse_obj(json_data, ('ResultSet', 'Result', ..., 'VideoUrlSet', 'VideoUrl', ...)) or []:
483 delivery = vid.get('delivery')
484 url = url_or_none(vid.get('Url'))
485 if not delivery or not url:
487 elif delivery == 'hls':
489 self._extract_m3u8_formats(
490 url, content_id, 'mp4', 'm3u8_native',
491 m3u8_id='hls', fatal=False))
495 'format_id': f'http-{vid.get("bitrate")}',
496 'height': int_or_none(vid.get('height')),
497 'width': int_or_none(vid.get('width')),
498 'tbr': int_or_none(vid.get('bitrate')),
500 self._remove_duplicate_formats(formats)
501 self._sort_formats(formats)
505 def _real_extract(self, url):
506 video_id = self._match_id(url)
507 webpage = self._download_webpage(url, video_id)
508 preloaded_state = self._search_json(r'__PRELOADED_STATE__\s*=', webpage, 'preloaded state', video_id)
510 content_id = traverse_obj(
511 preloaded_state, ('articleDetail', 'paragraphs', ..., 'objectItems', ..., 'video', 'vid'),
512 get_all=False, expected_type=int)
513 if content_id is None:
514 raise ExtractorError('This article does not contain a video', expected=True)
516 HOST = 'news.yahoo.co.jp'
517 space_id = traverse_obj(preloaded_state, ('pageData', 'spaceId'), expected_type=str)
518 json_data = self._download_json(
519 f'https://feapi-yvpub.yahooapis.jp/v1/content/{content_id}',
521 'appid': 'dj0zaiZpPVZMTVFJR0FwZWpiMyZzPWNvbnN1bWVyc2VjcmV0Jng9YjU-',
524 'ak': hashlib.md5('_'.join((space_id, HOST)).encode()).hexdigest() if space_id else '',
525 'device_type': '1100',
529 traverse_obj(preloaded_state,
530 ('articleDetail', 'headline'), ('pageData', 'pageParam', 'title'),
532 or self._html_search_meta(('og:title', 'twitter:title'), webpage, 'title', default=None)
533 or self._html_extract_title(webpage))
535 traverse_obj(preloaded_state, ('pageData', 'description'), expected_type=str)
536 or self._html_search_meta(
537 ('og:description', 'description', 'twitter:description'),
538 webpage, 'description', default=None))
540 traverse_obj(preloaded_state, ('pageData', 'ogpImage'), expected_type=str)
541 or self._og_search_thumbnail(webpage, default=None)
542 or self._html_search_meta('twitter:image', webpage, 'thumbnail', default=None))
547 'description': description,
548 'thumbnail': thumbnail,
549 'formats': self._extract_formats(json_data, video_id),