+import hashlib
+import hmac
import re
import time
-import hmac
-import binascii
-import hashlib
-
-from .once import OnceIE
from .adobepass import AdobePassIE
+from .once import OnceIE
+from ..networking import HEADRequest, Request
from ..utils import (
- determine_ext,
ExtractorError,
+ determine_ext,
+ find_xpath_attr,
float_or_none,
int_or_none,
+ mimetype2ext,
parse_qs,
- sanitized_Request,
+ traverse_obj,
unsmuggle_url,
+ update_url,
update_url_query,
+ urlhandle_detect_ext,
xpath_with_ns,
- mimetype2ext,
- find_xpath_attr,
)
default_ns = 'http://www.w3.org/2005/SMIL21/Language'
if exception.get('value') == 'GeoLocationBlocked':
self.raise_geo_restricted(error_element.attrib['abstract'])
elif error_element.attrib['src'].startswith(
- 'http://link.theplatform.%s/s/errorFiles/Unavailable.'
- % self._TP_TLD):
+ f'http://link.theplatform.{self._TP_TLD}/s/errorFiles/Unavailable.'):
raise ExtractorError(
error_element.attrib['abstract'], expected=True)
- smil_formats = self._parse_smil_formats(
+ smil_formats, subtitles = self._parse_smil_formats_and_subtitles(
meta, smil_url, video_id, namespace=default_ns,
# the parameters are from syfy.com, other sites may use others,
# they also work for nbc.com
formats.append(_format)
- subtitles = self._parse_smil_subtitles(meta, default_ns)
-
return formats, subtitles
def _download_theplatform_metadata(self, path, video_id):
- info_url = 'http://link.theplatform.%s/s/%s?format=preview' % (self._TP_TLD, path)
+ info_url = f'http://link.theplatform.{self._TP_TLD}/s/{path}?format=preview'
return self._download_json(info_url, video_id)
def _parse_theplatform_metadata(self, info):
_add_chapter(chapter.get('startTime'), chapter.get('endTime'))
_add_chapter(tp_chapters[-1].get('startTime'), tp_chapters[-1].get('endTime') or duration)
+ def extract_site_specific_field(field):
+ # A number of sites have custom-prefixed keys, e.g. 'cbc$seasonNumber'
+ return traverse_obj(info, lambda k, v: v and k.endswith(f'${field}'), get_all=False)
+
return {
'title': info['title'],
'subtitles': subtitles,
'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
'uploader': info.get('billingCode'),
'chapters': chapters,
+ 'creator': traverse_obj(info, ('author', {str})) or None,
+ 'categories': traverse_obj(info, (
+ 'categories', lambda _, v: v.get('label') in ('category', None), 'name', {str})) or None,
+ 'tags': traverse_obj(info, ('keywords', {lambda x: re.split(r'[;,]\s?', x) if x else None})),
+ 'location': extract_site_specific_field('region'),
+ 'series': extract_site_specific_field('show'),
+ 'season_number': int_or_none(extract_site_specific_field('seasonNumber')),
+ 'media_type': extract_site_specific_field('programmingType') or extract_site_specific_field('type'),
}
def _extract_theplatform_metadata(self, path, video_id):
<meta\s+
property=(["'])(?:og:video(?::(?:secure_)?url)?|twitter:player)\1\s+
content=(["'])(?P<url>https?://player\.theplatform\.com/p/.+?)\2''',
- r'(?s)<(?:iframe|script)[^>]+src=(["\'])(?P<url>(?:https?:)?//player\.theplatform\.com/p/.+?)\1'
+ r'(?s)<(?:iframe|script)[^>]+src=(["\'])(?P<url>(?:https?:)?//player\.theplatform\.com/p/.+?)\1',
]
_TESTS = [{
'params': {
# rtmp download
'skip_download': True,
- }
+ },
+ 'skip': 'CNet no longer uses ThePlatform',
}, {
'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
'info_dict': {
'description': 'md5:644ad9188d655b742f942bf2e06b002d',
'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
'uploader': 'EGSM',
- }
+ },
+ 'skip': 'Dead link',
}, {
'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
'only_matching': True,
'upload_date': '20150701',
'uploader': 'NBCU-NEWS',
},
+ 'skip': 'Error: Player PID "nbcNewsOffsite" is disabled',
}, {
# From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
# geo-restricted (US), HLS encrypted with AES-128
flags = '10' if include_qs else '00'
expiration_date = '%x' % (int(time.time()) + life)
- def str_to_hex(str):
- return binascii.b2a_hex(str.encode('ascii')).decode('ascii')
-
- def hex_to_bytes(hex):
- return binascii.a2b_hex(hex.encode('ascii'))
+ def str_to_hex(str_data):
+ return str_data.encode('ascii').hex()
relative_path = re.match(r'https?://link\.theplatform\.com/s/([^?]+)', url).group(1)
- clear_text = hex_to_bytes(flags + expiration_date + str_to_hex(relative_path))
+ clear_text = bytes.fromhex(flags + expiration_date + str_to_hex(relative_path))
checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
- return '%s&sig=%s' % (url, sig)
+ return f'{url}&sig={sig}'
def _real_extract(self, url):
url, smuggled_data = unsmuggle_url(url, {})
break
if feed_id is None:
raise ExtractorError('Unable to find feed id')
- return self.url_result('http://feed.theplatform.com/f/%s/%s?byGuid=%s' % (
+ return self.url_result('http://feed.theplatform.com/f/{}/{}?byGuid={}'.format(
provider_id, feed_id, qs_dict['guid'][0]))
if smuggled_data.get('force_smil_url', False):
source_url = smuggled_data.get('source_url')
if source_url:
headers['Referer'] = source_url
- request = sanitized_Request(url, headers=headers)
+ request = Request(url, headers=headers)
webpage = self._download_webpage(request, video_id)
smil_url = self._search_regex(
r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
config_url = config_url.replace('swf/', 'config/')
config_url = config_url.replace('onsite/', 'onsite/config/')
config = self._download_json(config_url, video_id, 'Downloading config')
- if 'releaseUrl' in config:
- release_url = config['releaseUrl']
- else:
- release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
+ release_url = config.get('releaseUrl') or f'http://link.theplatform.com/s/{path}?mbr=true'
smil_url = release_url + '&formats=MPEG4&manifest=f4m'
else:
- smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
+ smil_url = f'http://link.theplatform.com/s/{path}?mbr=true'
sig = smuggled_data.get('sig')
if sig:
formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
+ # With some sites, manifest URL must be forced to extract HLS formats
+ if not traverse_obj(formats, lambda _, v: v['format_id'].startswith('hls')):
+ m3u8_url = update_url(url, query='mbr=true&manifest=m3u', fragment=None)
+ urlh = self._request_webpage(
+ HEADRequest(m3u8_url), video_id, 'Checking for HLS formats', 'No HLS formats found', fatal=False)
+ if urlh and urlhandle_detect_ext(urlh) == 'm3u8':
+ m3u8_fmts, m3u8_subs = self._extract_m3u8_formats_and_subtitles(
+ m3u8_url, video_id, m3u8_id='hls', fatal=False)
+ formats.extend(m3u8_fmts)
+ self._merge_subtitles(m3u8_subs, target=subtitles)
+
ret = self._extract_theplatform_metadata(path, video_id)
combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
ret.update({
if asset_type in asset_types_query:
query.update(asset_types_query[asset_type])
cur_formats, cur_subtitles = self._extract_theplatform_smil(update_url_query(
- main_smil_url or smil_url, query), video_id, 'Downloading SMIL data for %s' % asset_type)
+ main_smil_url or smil_url, query), video_id, f'Downloading SMIL data for {asset_type}')
formats.extend(cur_formats)
subtitles = self._merge_subtitles(subtitles, cur_subtitles)
timestamp = int_or_none(entry.get('media$availableDate'), scale=1000)
categories = [item['media$name'] for item in entry.get('media$categories', [])]
- ret = self._extract_theplatform_metadata('%s/%s' % (provider_id, first_video_id), video_id)
+ ret = self._extract_theplatform_metadata(f'{provider_id}/{first_video_id}', video_id)
subtitles = self._merge_subtitles(subtitles, ret['subtitles'])
ret.update({
'id': video_id,
projects / yt-dlp.git / blobdiff