yt_dlp/extractor/piksel.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     dict_get,
   6     ExtractorError,
   7     int_or_none,
   8     join_nonempty,
   9     parse_iso8601,
  10     try_get,
  11     unescapeHTML,
  12 )
  13
  14
  15 class PikselIE(InfoExtractor):
  16     _VALID_URL = r'''(?x)https?://
  17         (?:
  18             (?:
  19                 player\.
  20                     (?:
  21                         olympusattelecom|
  22                         vibebyvista
  23                     )|
  24                 (?:api|player)\.multicastmedia|
  25                 (?:api-ovp|player)\.piksel
  26             )\.com|
  27             (?:
  28                 mz-edge\.stream\.co|
  29                 movie-s\.nhk\.or
  30             )\.jp|
  31             vidego\.baltimorecity\.gov
  32         )/v/(?:refid/(?P<refid>[^/]+)/prefid/)?(?P<id>[\w-]+)'''
  33     _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?:)?//player\.piksel\.com/v/[a-z0-9]+)']
  34     _TESTS = [
  35         {
  36             'url': 'http://player.piksel.com/v/ums2867l',
  37             'md5': '34e34c8d89dc2559976a6079db531e85',
  38             'info_dict': {
  39                 'id': 'ums2867l',
  40                 'ext': 'mp4',
  41                 'title': 'GX-005 with Caption',
  42                 'timestamp': 1481335659,
  43                 'upload_date': '20161210'
  44             }
  45         },
  46         {
  47             # Original source: http://www.uscourts.gov/cameras-courts/state-washington-vs-donald-j-trump-et-al
  48             'url': 'https://player.piksel.com/v/v80kqp41',
  49             'md5': '753ddcd8cc8e4fa2dda4b7be0e77744d',
  50             'info_dict': {
  51                 'id': 'v80kqp41',
  52                 'ext': 'mp4',
  53                 'title': 'WAW- State of Washington vs. Donald J. Trump, et al',
  54                 'description': 'State of Washington vs. Donald J. Trump, et al, Case Number 17-CV-00141-JLR, TRO Hearing, Civil Rights Case, 02/3/2017, 1:00 PM (PST), Seattle Federal Courthouse, Seattle, WA, Judge James L. Robart presiding.',
  55                 'timestamp': 1486171129,
  56                 'upload_date': '20170204'
  57             }
  58         },
  59         {
  60             # https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2019240/
  61             'url': 'http://player.piksel.com/v/refid/nhkworld/prefid/nw_vod_v_en_2019_240_20190823233000_02_1566873477',
  62             'only_matching': True,
  63         }
  64     ]
  65
  66     def _call_api(self, app_token, resource, display_id, query, fatal=True):
  67         response = (self._download_json(
  68             'http://player.piksel.com/ws/ws_%s/api/%s/mode/json/apiv/5' % (resource, app_token),
  69             display_id, query=query, fatal=fatal) or {}).get('response')
  70         failure = try_get(response, lambda x: x['failure']['reason'])
  71         if failure:
  72             if fatal:
  73                 raise ExtractorError(failure, expected=True)
  74             self.report_warning(failure)
  75         return response
  76
  77     def _real_extract(self, url):
  78         ref_id, display_id = self._match_valid_url(url).groups()
  79         webpage = self._download_webpage(url, display_id)
  80         app_token = self._search_regex([
  81             r'clientAPI\s*:\s*"([^"]+)"',
  82             r'data-de-api-key\s*=\s*"([^"]+)"'
  83         ], webpage, 'app token')
  84         query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
  85         program = self._call_api(
  86             app_token, 'program', display_id, query)['WsProgramResponse']['program']
  87         video_id = program['uuid']
  88         video_data = program['asset']
  89         title = video_data['title']
  90         asset_type = dict_get(video_data, ['assetType', 'asset_type'])
  91
  92         formats = []
  93
  94         def process_asset_file(asset_file):
  95             if not asset_file:
  96                 return
  97             # TODO: extract rtmp formats
  98             http_url = asset_file.get('http_url')
  99             if not http_url:
 100                 return
 101             tbr = None
 102             vbr = int_or_none(asset_file.get('videoBitrate'), 1024)
 103             abr = int_or_none(asset_file.get('audioBitrate'), 1024)
 104             if asset_type == 'video':
 105                 tbr = vbr + abr
 106             elif asset_type == 'audio':
 107                 tbr = abr
 108
 109             formats.append({
 110                 'format_id': join_nonempty('http', tbr),
 111                 'url': unescapeHTML(http_url),
 112                 'vbr': vbr,
 113                 'abr': abr,
 114                 'width': int_or_none(asset_file.get('videoWidth')),
 115                 'height': int_or_none(asset_file.get('videoHeight')),
 116                 'filesize': int_or_none(asset_file.get('filesize')),
 117                 'tbr': tbr,
 118             })
 119
 120         def process_asset_files(asset_files):
 121             for asset_file in (asset_files or []):
 122                 process_asset_file(asset_file)
 123
 124         process_asset_files(video_data.get('assetFiles'))
 125         process_asset_file(video_data.get('referenceFile'))
 126         if not formats:
 127             asset_id = video_data.get('assetid') or program.get('assetid')
 128             if asset_id:
 129                 process_asset_files(try_get(self._call_api(
 130                     app_token, 'asset_file', display_id, {
 131                         'assetid': asset_id,
 132                     }, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
 133
 134         m3u8_url = dict_get(video_data, [
 135             'm3u8iPadURL',
 136             'ipadM3u8Url',
 137             'm3u8AndroidURL',
 138             'm3u8iPhoneURL',
 139             'iphoneM3u8Url'])
 140         if m3u8_url:
 141             formats.extend(self._extract_m3u8_formats(
 142                 m3u8_url, video_id, 'mp4', 'm3u8_native',
 143                 m3u8_id='hls', fatal=False))
 144
 145         smil_url = dict_get(video_data, ['httpSmil', 'hdSmil', 'rtmpSmil'])
 146         if smil_url:
 147             transform_source = None
 148             if ref_id == 'nhkworld':
 149                 # TODO: figure out if this is something to be fixed in urljoin,
 150                 # _parse_smil_formats or keep it here
 151                 transform_source = lambda x: x.replace('src="/', 'src="').replace('/media"', '/media/"')
 152             formats.extend(self._extract_smil_formats(
 153                 re.sub(r'/od/[^/]+/', '/od/http/', smil_url), video_id,
 154                 transform_source=transform_source, fatal=False))
 155
 156         self._sort_formats(formats, ('tbr', ))  # Incomplete resolution information
 157
 158         subtitles = {}
 159         for caption in video_data.get('captions', []):
 160             caption_url = caption.get('url')
 161             if caption_url:
 162                 subtitles.setdefault(caption.get('locale', 'en'), []).append({
 163                     'url': caption_url})
 164
 165         return {
 166             'id': video_id,
 167             'title': title,
 168             'description': video_data.get('description'),
 169             'thumbnail': video_data.get('thumbnailUrl'),
 170             'timestamp': parse_iso8601(video_data.get('dateadd')),
 171             'formats': formats,
 172             'subtitles': subtitles,
 173         }