yt_dlp/extractor/piksel.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     ExtractorError,
   6     dict_get,
   7     int_or_none,
   8     join_nonempty,
   9     parse_iso8601,
  10     traverse_obj,
  11     try_get,
  12     unescapeHTML,
  13     urljoin,
  14 )
  15
  16
  17 class PikselIE(InfoExtractor):
  18     _VALID_URL = r'''(?x)https?://
  19         (?:
  20             (?:
  21                 player\.
  22                     (?:
  23                         olympusattelecom|
  24                         vibebyvista
  25                     )|
  26                 (?:api|player)\.multicastmedia|
  27                 (?:api-ovp|player)\.piksel
  28             )\.com|
  29             (?:
  30                 mz-edge\.stream\.co|
  31                 movie-s\.nhk\.or
  32             )\.jp|
  33             vidego\.baltimorecity\.gov
  34         )/v/(?:refid/(?P<refid>[^/]+)/prefid/)?(?P<id>[\w-]+)'''
  35     _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?:)?//player\.piksel\.com/v/[a-z0-9]+)']
  36     _TESTS = [
  37         {
  38             'url': 'http://player.piksel.com/v/ums2867l',
  39             'md5': '34e34c8d89dc2559976a6079db531e85',
  40             'info_dict': {
  41                 'id': 'ums2867l',
  42                 'ext': 'mp4',
  43                 'title': 'GX-005 with Caption',
  44                 'timestamp': 1481335659,
  45                 'upload_date': '20161210'
  46             }
  47         },
  48         {
  49             # Original source: http://www.uscourts.gov/cameras-courts/state-washington-vs-donald-j-trump-et-al
  50             'url': 'https://player.piksel.com/v/v80kqp41',
  51             'md5': '753ddcd8cc8e4fa2dda4b7be0e77744d',
  52             'info_dict': {
  53                 'id': 'v80kqp41',
  54                 'ext': 'mp4',
  55                 'title': 'WAW- State of Washington vs. Donald J. Trump, et al',
  56                 'description': 'State of Washington vs. Donald J. Trump, et al, Case Number 17-CV-00141-JLR, TRO Hearing, Civil Rights Case, 02/3/2017, 1:00 PM (PST), Seattle Federal Courthouse, Seattle, WA, Judge James L. Robart presiding.',
  57                 'timestamp': 1486171129,
  58                 'upload_date': '20170204'
  59             }
  60         },
  61         {
  62             # https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2019240/
  63             'url': 'http://player.piksel.com/v/refid/nhkworld/prefid/nw_vod_v_en_2019_240_20190823233000_02_1566873477',
  64             'only_matching': True,
  65         }
  66     ]
  67
  68     def _call_api(self, app_token, resource, display_id, query, host='https://player.piksel.com', fatal=True):
  69         url = urljoin(host, f'/ws/ws_{resource}/api/{app_token}/mode/json/apiv/5')
  70         response = traverse_obj(
  71             self._download_json(url, display_id, query=query, fatal=fatal), ('response', {dict})) or {}
  72         failure = traverse_obj(response, ('failure', 'reason')) if response else 'Empty response from API'
  73         if failure:
  74             if fatal:
  75                 raise ExtractorError(failure, expected=True)
  76             self.report_warning(failure)
  77         return response
  78
  79     def _real_extract(self, url):
  80         ref_id, display_id = self._match_valid_url(url).groups()
  81         webpage = self._download_webpage(url, display_id)
  82         app_token = self._search_regex([
  83             r'clientAPI\s*:\s*"([^"]+)"',
  84             r'data-de-api-key\s*=\s*"([^"]+)"'
  85         ], webpage, 'app token')
  86         query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
  87         program = self._call_api(
  88             app_token, 'program', display_id, query, url)['WsProgramResponse']['program']
  89         video_id = program['uuid']
  90         video_data = program['asset']
  91         title = video_data['title']
  92         asset_type = dict_get(video_data, ['assetType', 'asset_type'])
  93
  94         formats = []
  95
  96         def process_asset_file(asset_file):
  97             if not asset_file:
  98                 return
  99             # TODO: extract rtmp formats
 100             http_url = asset_file.get('http_url')
 101             if not http_url:
 102                 return
 103             tbr = None
 104             vbr = int_or_none(asset_file.get('videoBitrate'), 1024)
 105             abr = int_or_none(asset_file.get('audioBitrate'), 1024)
 106             if asset_type == 'video':
 107                 tbr = vbr + abr
 108             elif asset_type == 'audio':
 109                 tbr = abr
 110
 111             formats.append({
 112                 'format_id': join_nonempty('http', tbr),
 113                 'url': unescapeHTML(http_url),
 114                 'vbr': vbr,
 115                 'abr': abr,
 116                 'width': int_or_none(asset_file.get('videoWidth')),
 117                 'height': int_or_none(asset_file.get('videoHeight')),
 118                 'filesize': int_or_none(asset_file.get('filesize')),
 119                 'tbr': tbr,
 120             })
 121
 122         def process_asset_files(asset_files):
 123             for asset_file in (asset_files or []):
 124                 process_asset_file(asset_file)
 125
 126         process_asset_files(video_data.get('assetFiles'))
 127         process_asset_file(video_data.get('referenceFile'))
 128         if not formats:
 129             asset_id = video_data.get('assetid') or program.get('assetid')
 130             if asset_id:
 131                 process_asset_files(try_get(self._call_api(
 132                     app_token, 'asset_file', display_id, {
 133                         'assetid': asset_id,
 134                     }, url, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
 135
 136         m3u8_url = dict_get(video_data, [
 137             'm3u8iPadURL',
 138             'ipadM3u8Url',
 139             'm3u8AndroidURL',
 140             'm3u8iPhoneURL',
 141             'iphoneM3u8Url'])
 142         if m3u8_url:
 143             formats.extend(self._extract_m3u8_formats(
 144                 m3u8_url, video_id, 'mp4', 'm3u8_native',
 145                 m3u8_id='hls', fatal=False))
 146
 147         smil_url = dict_get(video_data, ['httpSmil', 'hdSmil', 'rtmpSmil'])
 148         if smil_url:
 149             transform_source = None
 150             if ref_id == 'nhkworld':
 151                 # TODO: figure out if this is something to be fixed in urljoin,
 152                 # _parse_smil_formats or keep it here
 153                 transform_source = lambda x: x.replace('src="/', 'src="').replace('/media"', '/media/"')
 154             formats.extend(self._extract_smil_formats(
 155                 re.sub(r'/od/[^/]+/', '/od/http/', smil_url), video_id,
 156                 transform_source=transform_source, fatal=False))
 157
 158         subtitles = {}
 159         for caption in video_data.get('captions', []):
 160             caption_url = caption.get('url')
 161             if caption_url:
 162                 subtitles.setdefault(caption.get('locale', 'en'), []).append({
 163                     'url': caption_url})
 164
 165         return {
 166             'id': video_id,
 167             'title': title,
 168             'description': video_data.get('description'),
 169             'thumbnail': video_data.get('thumbnailUrl'),
 170             'timestamp': parse_iso8601(video_data.get('dateadd')),
 171             'formats': formats,
 172             'subtitles': subtitles,
 173             '_format_sort_fields': ('tbr', ),  # Incomplete resolution information
 174         }