5 from .brightcove
import BrightcoveNewIE
6 from .common
import InfoExtractor
, SearchInfoExtractor
7 from .youtube
import YoutubeIE
21 class YahooIE(InfoExtractor
):
22 IE_DESC
= 'Yahoo screen and movies'
23 _VALID_URL
= r
'(?P<url>https?://(?:(?P<country>[a-zA-Z]{2}(?:-[a-zA-Z]{2})?|malaysia)\.)?(?:[\da-zA-Z_-]+\.)?yahoo\.com/(?:[^/]+/)*(?P<id>[^?&#]*-[0-9]+(?:-[a-z]+)?)\.html)'
24 _EMBED_REGEX
= [r
'<iframe[^>]+?src=(["\'])(?P
<url
>https?
://(?
:screen|movies
)\
.yahoo\
.com
/.+?\
.html
\?format
=embed
)\
1']
27 'url
': 'http
://screen
.yahoo
.com
/julian
-smith
-travis
-legg
-watch
-214727115.html
',
29 'id': '2d25e626
-2378-391f
-ada0
-ddaf1417e588
',
31 'title
': 'Julian Smith
& Travis Legg Watch Julian Smith
',
32 'description
': 'Julian
and Travis watch Julian Smith
',
34 'timestamp
': 1369812016,
35 'upload_date
': '20130529',
37 'skip
': 'No longer exists
',
39 'url
': 'https
://screen
.yahoo
.com
/community
/community
-sizzle
-reel
-203225340.html?format
=embed
',
40 'md5
': '7993e572fac98e044588d0b5260f4352
',
42 'id': '4fe78544
-8d48
-39d8
-97cd
-13f205d9fcdb
',
44 'title
': "Yahoo Saves 'Community
'",
45 'description
': 'md5
:4d4145af2fd3de00cbb6c1d664105053
',
47 'timestamp
': 1406838636,
48 'upload_date
': '20140731',
50 'skip
': 'Unfortunately
, this video
is not available
in your region
',
52 'url
': 'https
://uk
.screen
.yahoo
.com
/editor
-picks
/cute
-raccoon
-freed
-drain
-using
-091756545.html
',
53 'md5
': '71298482f7c64cbb7fa064e4553ff1c1
',
55 'id': 'b3affa53
-2e14
-3590-852b
-0e0db6cd1a58
',
57 'title
': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder
',
58 'description
': 'md5
:f66c890e1490f4910a9953c941dee944
',
60 'timestamp
': 1414489862,
61 'upload_date
': '20141028',
63 'skip
': 'No longer exists
',
65 'url
': 'http
://news
.yahoo
.com
/video
/china
-moses
-crazy
-blues
-104538833.html
',
66 'md5
': '88e209b417f173d86186bef6e4d1f160
',
68 'id': 'f885cf7f
-43d4
-3450-9fac
-46ac30ece521
',
70 'title
': 'China Moses Is Crazy About the Blues
',
71 'description
': 'md5
:9900ab8cd5808175c7b3fe55b979bed0
',
73 'timestamp
': 1385722202,
74 'upload_date
': '20131129',
77 'url
': 'https
://www
.yahoo
.com
/movies
/v
/true
-story
-trailer
-173000497.html
',
78 'md5
': '2a9752f74cb898af5d1083ea9f661b58
',
80 'id': '071c4013
-ce30
-3a93
-a5b2
-e0413cd4a9d1
',
82 'title
': '\'True Story
\' Trailer
',
83 'description
': 'True Story
',
85 'timestamp
': 1418919206,
86 'upload_date
': '20141218',
89 'url
': 'https
://gma
.yahoo
.com
/pizza
-delivery
-man
-surprised
-huge
-tip
-college
-kids
-195200785.html
',
90 'only_matching
': True,
92 'note
': 'NBC Sports embeds
',
93 'url
': 'http
://sports
.yahoo
.com
/blogs
/ncaab
-the
-dagger
/tyler
-kalinoski
-s
-buzzer
-beater
-caps
-davidson
-s
-comeback
-win
-185609842.html?guid
=nbc_cbk_davidsonbuzzerbeater_150313
',
97 'description
': 'md5
:df390f70a9ba7c95ff1daace988f0d8d
',
98 'title
': 'Tyler Kalinoski hits buzzer
-beater to lift Davidson
',
99 'upload_date
': '20150313',
100 'uploader
': 'NBCU
-SPORTS
',
101 'timestamp
': 1426270238,
104 'url
': 'https
://tw
.news
.yahoo
.com
/-100120367.html
',
105 'only_matching
': True,
107 # Query result is embedded in webpage, but explicit request to video API fails with geo restriction
108 'url
': 'https
://screen
.yahoo
.com
/community
/communitary
-community
-episode
-1-ladders
-154501237.html
',
109 'md5
': '4fbafb9c9b6f07aa8f870629f6671b35
',
111 'id': '1f32853c
-a271
-3eef
-8cb6
-f6d6872cb504
',
113 'title
': 'Communitary
- Community Episode
1: Ladders
',
114 'description
': 'md5
:8fc39608213295748e1e289807838c97
',
116 'timestamp
': 1440436550,
117 'upload_date
': '20150824',
118 'series
': 'Communitary
',
122 'skip
': 'No longer exists
',
125 'url
': 'https
://tw
.video
.yahoo
.com
/movie
-tw
/單車天使
-中文版預
-092316541.html
',
127 'id': 'ba133ff2
-0793-3510-b636
-59dfe9ff6cff
',
129 'title
': '單車天使
- 中文版預
',
130 'description
': '中文版預
',
131 'timestamp
': 1476696196,
132 'upload_date
': '20161017',
135 'skip_download
': True,
138 # Contains both a Yahoo hosted video and multiple Youtube embeds
139 'url
': 'https
://www
.yahoo
.com
/entertainment
/gwen
-stefani
-reveals
-the
-pop
-hit
-she
-passed
-on
-assigns
-it
-to
-her
-voice
-contestant
-instead
-033045672.html
',
141 'id': '46c5d95a
-528f
-3d03
-b732
-732fcadd51de
',
142 'title
': 'Gwen Stefani reveals the pop hit she passed on
, assigns it to her
\'Voice
\' contestant instead
',
143 'description
': 'Gwen decided
not to record this hit herself
, but she decided it was the perfect fit
for Kyndall Inskeep
.',
147 'id': '966d4262
-4fd1
-3aaa
-b45b
-049ca6e38ba6
',
149 'title
': 'Gwen Stefani reveals she turned down one of Sia
\'s best songs
',
150 'description
': 'On
"The Voice" Tuesday
, Gwen Stefani told Taylor Swift which Sia hit was almost hers
.',
151 'timestamp
': 1572406500,
152 'upload_date
': '20191030',
158 'title
': 'Kyndal Inskeep
"Performs the Hell Out of" Sia
\'s
"Elastic Heart" - The Voice Knockouts
2019',
159 'description
': 'md5
:7fe8e3d5806f96002e55f190d1d94479
',
160 'uploader
': 'The Voice
',
161 'uploader_id
': 'NBCTheVoice
',
162 'upload_date
': '20191029',
168 'expected_warnings
': ['HTTP Error
404', 'Ignoring subtitle tracks
'],
170 'url
': 'https
://malaysia
.news
.yahoo
.com
/video
/bystanders
-help-ontario
-policeman
-bust
-190932818.html
',
171 'only_matching
': True,
173 'url
': 'https
://es
-us
.noticias
.yahoo
.com
/es
-la
-puerta
-irrompible
-que
-110539379.html
',
174 'only_matching
': True,
176 'url
': 'https
://www
.yahoo
.com
/entertainment
/v
/longtime
-cbs
-news
-60-minutes
-032036500-cbs
.html
',
177 'only_matching
': True,
180 def _extract_yahoo_video(self, video_id, country):
181 video = self._download_json(
182 'https
://%s.yahoo
.com
/_td
/api
/resource
/VideoService
.videos
;view
=full
;video_ids
=["%s"]' % (country, video_id),
183 video_id, 'Downloading video JSON metadata
')[0]
184 title = video['title
']
186 if country == 'malaysia
':
189 is_live = video.get('live_state
') == 'live
'
190 fmts = ('m3u8
',) if is_live else ('webm
', 'mp4
')
196 media_obj = self._download_json(
197 'https
://video
-api
.yql
.yahoo
.com
/v1
/video
/sapi
/streams
/' + video_id,
198 video_id, 'Downloading
%s JSON metadata
' % fmt,
199 headers=self.geo_verification_headers(), query={
201 'region
': country.upper(),
202 })['query
']['results
']['mediaObj
'][0]
203 msg = media_obj.get('status
', {}).get('msg
')
205 for s in media_obj.get('streams
', []):
208 if not host or not path:
211 if s.get('format
') == 'm3u8
':
212 formats.extend(self._extract_m3u8_formats(
213 s_url, video_id, 'mp4
', m3u8_id='hls
', fatal=False))
215 tbr = int_or_none(s.get('bitrate
'))
218 'format_id
': fmt + ('-%d' % tbr if tbr else ''),
219 'width
': int_or_none(s.get('width
')),
220 'height
': int_or_none(s.get('height
')),
222 'fps
': int_or_none(s.get('framerate
')),
225 for cc in media_obj.get('closedcaptions
', []):
226 cc_url = cc.get('url
')
227 if not cc_url or cc_url in urls:
230 subtitles.setdefault(cc.get('lang
') or 'en
-US
', []).append({
232 'ext
': mimetype2ext(cc.get('content_type
')),
235 streaming_url = video.get('streaming_url
')
236 if streaming_url and not is_live:
237 formats.extend(self._extract_m3u8_formats(
238 streaming_url, video_id, 'mp4
',
239 'm3u8_native
', m3u8_id='hls
', fatal=False))
241 if not formats and msg == 'geo restricted
':
242 self.raise_geo_restricted(metadata_available=True)
245 for thumb in video.get('thumbnails
', []):
246 thumb_url = thumb.get('url
')
250 'id': thumb.get('tag
'),
251 'url
': thumb.get('url
'),
252 'width
': int_or_none(thumb.get('width
')),
253 'height
': int_or_none(thumb.get('height
')),
256 series_info = video.get('series_info
') or {}
262 'thumbnails
': thumbnails,
263 'description
': clean_html(video.get('description
')),
264 'timestamp
': parse_iso8601(video.get('publish_time
')),
265 'subtitles
': subtitles,
266 'duration
': int_or_none(video.get('duration
')),
267 'view_count
': int_or_none(video.get('view_count
')),
269 'series
': video.get('show_name
'),
270 'season_number
': int_or_none(series_info.get('season_number
')),
271 'episode_number
': int_or_none(series_info.get('episode_number
')),
274 def _real_extract(self, url):
275 url, country, display_id = self._match_valid_url(url).groups()
279 country = country.split('-')[0]
281 items = self._download_json(
282 'https
://%s.yahoo
.com
/caas
/content
/article
' % country, display_id,
283 'Downloading content JSON metadata
', query={
287 item = items['data
']['partnerData
']
288 if item.get('type') != 'video
':
291 cover = item.get('cover
') or {}
292 if cover.get('type') == 'yvideo
':
293 cover_url = cover.get('url
')
295 entries.append(self.url_result(
296 cover_url, 'Yahoo
', cover.get('uuid
')))
298 for e in (item.get('body
') or []):
299 if e.get('type') == 'videoIframe
':
300 iframe_url = e.get('url
')
302 entries.append(self.url_result(iframe_url))
304 if item.get('type') == 'storywithleadvideo
':
305 iframe_url = try_get(item, lambda x: x['meta
']['player
']['url
'])
307 entries.append(self.url_result(iframe_url))
309 self.report_warning("Yahoo didn't provide an iframe url
for this storywithleadvideo
")
311 if items.get('markup'):
313 self.url_result(yt_url) for yt_url in YoutubeIE._extract_embed_urls(url, items['markup']))
315 return self.playlist_result(
316 entries, item.get('uuid'),
317 item.get('title'), item.get('summary'))
319 info = self._extract_yahoo_video(item['uuid'], country)
320 info['display_id'] = display_id
324 class YahooSearchIE(SearchInfoExtractor):
325 IE_DESC = 'Yahoo screen search'
327 IE_NAME = 'screen.yahoo:search'
328 _SEARCH_KEY = 'yvsearch'
330 def _search_results(self, query):
331 for pagenum in itertools.count(0):
332 result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (urllib.parse.quote_plus(query), pagenum * 30)
333 info = self._download_json(result_url, query,
334 note='Downloading results page ' + str(pagenum + 1))
335 yield from (self.url_result(result['rurl']) for result in info['results'])
336 if info['m']['last'] >= info['m']['total'] - 1:
340 class YahooGyaOPlayerIE(InfoExtractor):
341 IE_NAME = 'yahoo:gyao:player'
342 _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:player|episode(?:/[^/]+)?)|streaming\.yahoo\.co\.jp/c/y)/(?P<id>\d+/v\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
344 'url': 'https://gyao.yahoo.co.jp/player/00998/v00818/v0000000000000008564/',
346 'id': '5993125228001',
348 'title': 'フューリー 【字幕版】',
349 'description': 'md5:21e691c798a15330eda4db17a8fe45a5',
350 'uploader_id': '4235717419001',
351 'upload_date': '20190124',
352 'timestamp': 1548294365,
356 'skip_download': True,
359 'url': 'https://streaming.yahoo.co.jp/c/y/01034/v00133/v0000000000000000706/',
360 'only_matching': True,
362 'url': 'https://gyao.yahoo.co.jp/episode/%E3%81%8D%E3%81%AE%E3%81%86%E4%BD%95%E9%A3%9F%E3%81%B9%E3%81%9F%EF%BC%9F%20%E7%AC%AC2%E8%A9%B1%202019%2F4%2F12%E6%94%BE%E9%80%81%E5%88%86/5cb02352-b725-409e-9f8d-88f947a9f682',
363 'only_matching': True,
365 'url': 'https://gyao.yahoo.co.jp/episode/5fa1226c-ef8d-4e93-af7a-fd92f4e30597',
366 'only_matching': True,
370 def _real_extract(self, url):
371 video_id = self._match_id(url).replace('/', ':')
372 headers = self.geo_verification_headers()
373 headers['Accept'] = 'application/json'
374 resp = self._download_json(
375 'https://gyao.yahoo.co.jp/apis/playback/graphql', video_id, query={
376 'appId': 'dj00aiZpPUNJeDh2cU1RazU3UCZzPWNvbnN1bWVyc2VjcmV0Jng9NTk-',
378 content(parameter: {contentId: "%s", logicaAgent: PC_WEB}) {
388 content = resp['data']['content']
390 msg = resp['errors'][0]['message']
391 if msg == 'not in japan':
392 self.raise_geo_restricted(countries=['JP'])
393 raise ExtractorError(msg)
394 video = content['video']
396 '_type': 'url_transparent',
398 'title': video['title'],
400 'http://players.brightcove.net/4235717419001/SyG5P0gjb_default/index.html?videoId=' + video['delivery']['id'],
401 {'geo_countries': ['JP']}),
402 'ie_key': BrightcoveNewIE.ie_key(),
406 class YahooGyaOIE(InfoExtractor):
407 IE_NAME = 'yahoo:gyao'
408 _VALID_URL = r'https?://(?:gyao\.yahoo\.co\.jp/(?:p|title(?:/[^/]+)?)|streaming\.yahoo\.co\.jp/p/y)/(?P<id>\d+/v\d+|[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
410 'url': 'https://gyao.yahoo.co.jp/title/%E3%82%BF%E3%82%A4%E3%83%A0%E3%83%9C%E3%82%AB%E3%83%B3%E3%82%B7%E3%83%AA%E3%83%BC%E3%82%BA%20%E3%83%A4%E3%83%83%E3%82%BF%E3%83%BC%E3%83%9E%E3%83%B3/5f60ceb3-6e5e-40ef-ba40-d68b598d067f',
412 'id': '5f60ceb3-6e5e-40ef-ba40-d68b598d067f',
414 'playlist_mincount': 80,
416 'url': 'https://gyao.yahoo.co.jp/p/00449/v03102/',
417 'only_matching': True,
419 'url': 'https://streaming.yahoo.co.jp/p/y/01034/v00133/',
420 'only_matching': True,
422 'url': 'https://gyao.yahoo.co.jp/title/%E3%81%97%E3%82%83%E3%81%B9%E3%81%8F%E3%82%8A007/5b025a49-b2e5-4dc7-945c-09c6634afacf',
423 'only_matching': True,
425 'url': 'https://gyao.yahoo.co.jp/title/5b025a49-b2e5-4dc7-945c-09c6634afacf',
426 'only_matching': True,
429 def _entries(self, program_id):
432 playlist = self._download_json(
433 f'https://gyao.yahoo.co.jp/api/programs/{program_id}/videos?page={page}&serviceId=gy', program_id,
434 note=f'Downloading JSON metadata page {page}')
437 for video in playlist['videos']:
438 video_id = video.get('id')
441 if video.get('streamingAvailability') == 'notYet':
443 yield self.url_result(
444 'https://gyao.yahoo.co.jp/player/%s/' % video_id.replace(':', '/'),
445 YahooGyaOPlayerIE.ie_key(), video_id)
446 if playlist.get('ended'):
450 def _real_extract(self, url):
451 program_id = self._match_id(url).replace('/', ':')
452 return self.playlist_result(self._entries(program_id), program_id)
455 class YahooJapanNewsIE(InfoExtractor):
456 IE_NAME = 'yahoo:japannews'
457 IE_DESC = 'Yahoo! Japan News'
458 _VALID_URL = r'https?://news\.yahoo\.co\.jp/(?:articles|feature)/(?P<id>[a-zA-Z0-9]+)'
459 _GEO_COUNTRIES = ['JP']
461 'url': 'https://news.yahoo.co.jp/articles/a70fe3a064f1cfec937e2252c7fc6c1ba3201c0e',
463 'id': 'a70fe3a064f1cfec937e2252c7fc6c1ba3201c0e',
465 'title': '【独自】安倍元総理「国葬」中止求め“脅迫メール”…「子ども誘拐」“送信者”を追跡',
466 'description': 'md5:1c06974575f930f692d8696fbcfdc546',
467 'thumbnail': r're:https://.+',
470 'skip_download': True,
473 'url': 'https://news.yahoo.co.jp/feature/1356',
474 'only_matching': True
477 def _extract_formats(self, json_data, content_id):
480 for vid in traverse_obj(json_data, ('ResultSet', 'Result', ..., 'VideoUrlSet', 'VideoUrl', ...)) or []:
481 delivery = vid.get('delivery')
482 url = url_or_none(vid.get('Url'))
483 if not delivery or not url:
485 elif delivery == 'hls':
487 self._extract_m3u8_formats(
488 url, content_id, 'mp4', 'm3u8_native',
489 m3u8_id='hls', fatal=False))
493 'format_id': f'http-{vid.get("bitrate")}',
494 'height': int_or_none(vid.get('height')),
495 'width': int_or_none(vid.get('width')),
496 'tbr': int_or_none(vid.get('bitrate')),
498 self._remove_duplicate_formats(formats)
502 def _real_extract(self, url):
503 video_id = self._match_id(url)
504 webpage = self._download_webpage(url, video_id)
505 preloaded_state = self._search_json(r'__PRELOADED_STATE__\s*=', webpage, 'preloaded state', video_id)
507 content_id = traverse_obj(
508 preloaded_state, ('articleDetail', 'paragraphs', ..., 'objectItems', ..., 'video', 'vid'),
509 get_all=False, expected_type=int)
510 if content_id is None:
511 raise ExtractorError('This article does not contain a video', expected=True)
513 HOST = 'news.yahoo.co.jp'
514 space_id = traverse_obj(preloaded_state, ('pageData', 'spaceId'), expected_type=str)
515 json_data = self._download_json(
516 f'https://feapi-yvpub.yahooapis.jp/v1/content/{content_id}',
518 'appid': 'dj0zaiZpPVZMTVFJR0FwZWpiMyZzPWNvbnN1bWVyc2VjcmV0Jng9YjU-',
521 'ak': hashlib.md5('_'.join((space_id, HOST)).encode()).hexdigest() if space_id else '',
522 'device_type': '1100',
526 traverse_obj(preloaded_state,
527 ('articleDetail', 'headline'), ('pageData', 'pageParam', 'title'),
529 or self._html_search_meta(('og:title', 'twitter:title'), webpage, 'title', default=None)
530 or self._html_extract_title(webpage))
532 traverse_obj(preloaded_state, ('pageData', 'description'), expected_type=str)
533 or self._html_search_meta(
534 ('og:description', 'description', 'twitter:description'),
535 webpage, 'description', default=None))
537 traverse_obj(preloaded_state, ('pageData', 'ogpImage'), expected_type=str)
538 or self._og_search_thumbnail(webpage, default=None)
539 or self._html_search_meta('twitter:image', webpage, 'thumbnail', default=None))
544 'description': description,
545 'thumbnail': thumbnail,
546 'formats': self._extract_formats(json_data, video_id),