yt_dlp/extractor/kinja.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..compat import (
   5     compat_str,
   6     compat_urllib_parse_unquote,
   7 )
   8 from ..utils import (
   9     int_or_none,
  10     parse_iso8601,
  11     strip_or_none,
  12     try_get,
  13     unescapeHTML,
  14     urljoin,
  15 )
  16
  17
  18 class KinjaEmbedIE(InfoExtractor):
  19     IENAME = 'kinja:embed'
  20     _DOMAIN_REGEX = r'''(?:[^.]+\.)?
  21         (?:
  22             avclub|
  23             clickhole|
  24             deadspin|
  25             gizmodo|
  26             jalopnik|
  27             jezebel|
  28             kinja|
  29             kotaku|
  30             lifehacker|
  31             splinternews|
  32             the(?:inventory|onion|root|takeout)
  33         )\.com'''
  34     _COMMON_REGEX = r'''/
  35         (?:
  36             ajax/inset|
  37             embed/video
  38         )/iframe\?.*?\bid='''
  39     _VALID_URL = r'''(?x)https?://%s%s
  40         (?P<type>
  41             fb|
  42             imgur|
  43             instagram|
  44             jwp(?:layer)?-video|
  45             kinjavideo|
  46             mcp|
  47             megaphone|
  48             ooyala|
  49             soundcloud(?:-playlist)?|
  50             tumblr-post|
  51             twitch-stream|
  52             twitter|
  53             ustream-channel|
  54             vimeo|
  55             vine|
  56             youtube-(?:list|video)
  57         )-(?P<id>[^&]+)''' % (_DOMAIN_REGEX, _COMMON_REGEX)
  58     _TESTS = [{
  59         'url': 'https://kinja.com/ajax/inset/iframe?id=fb-10103303356633621',
  60         'only_matching': True,
  61     }, {
  62         'url': 'https://kinja.com/ajax/inset/iframe?id=kinjavideo-100313',
  63         'only_matching': True,
  64     }, {
  65         'url': 'https://kinja.com/ajax/inset/iframe?id=megaphone-PPY1300931075',
  66         'only_matching': True,
  67     }, {
  68         'url': 'https://kinja.com/ajax/inset/iframe?id=ooyala-xzMXhleDpopuT0u1ijt_qZj3Va-34pEX%2FZTIxYmJjZDM2NWYzZDViZGRiOWJjYzc5',
  69         'only_matching': True,
  70     }, {
  71         'url': 'https://kinja.com/ajax/inset/iframe?id=soundcloud-128574047',
  72         'only_matching': True,
  73     }, {
  74         'url': 'https://kinja.com/ajax/inset/iframe?id=soundcloud-playlist-317413750',
  75         'only_matching': True,
  76     }, {
  77         'url': 'https://kinja.com/ajax/inset/iframe?id=tumblr-post-160130699814-daydreams-at-midnight',
  78         'only_matching': True,
  79     }, {
  80         'url': 'https://kinja.com/ajax/inset/iframe?id=twitch-stream-libratus_extra',
  81         'only_matching': True,
  82     }, {
  83         'url': 'https://kinja.com/ajax/inset/iframe?id=twitter-1068875942473404422',
  84         'only_matching': True,
  85     }, {
  86         'url': 'https://kinja.com/ajax/inset/iframe?id=ustream-channel-10414700',
  87         'only_matching': True,
  88     }, {
  89         'url': 'https://kinja.com/ajax/inset/iframe?id=vimeo-120153502',
  90         'only_matching': True,
  91     }, {
  92         'url': 'https://kinja.com/ajax/inset/iframe?id=vine-5BlvV5qqPrD',
  93         'only_matching': True,
  94     }, {
  95         'url': 'https://kinja.com/ajax/inset/iframe?id=youtube-list-BCQ3KyrPjgA/PLE6509247C270A72E',
  96         'only_matching': True,
  97     }, {
  98         'url': 'https://kinja.com/ajax/inset/iframe?id=youtube-video-00QyL0AgPAE',
  99         'only_matching': True,
 100     }]
 101     _JWPLATFORM_PROVIDER = ('cdn.jwplayer.com/v2/media/', 'JWPlatform')
 102     _PROVIDER_MAP = {
 103         'fb': ('facebook.com/video.php?v=', 'Facebook'),
 104         'imgur': ('imgur.com/', 'Imgur'),
 105         'instagram': ('instagram.com/p/', 'Instagram'),
 106         'jwplayer-video': _JWPLATFORM_PROVIDER,
 107         'jwp-video': _JWPLATFORM_PROVIDER,
 108         'megaphone': ('player.megaphone.fm/', 'Generic'),
 109         'ooyala': ('player.ooyala.com/player.js?embedCode=', 'Ooyala'),
 110         'soundcloud': ('api.soundcloud.com/tracks/', 'Soundcloud'),
 111         'soundcloud-playlist': ('api.soundcloud.com/playlists/', 'SoundcloudPlaylist'),
 112         'tumblr-post': ('%s.tumblr.com/post/%s', 'Tumblr'),
 113         'twitch-stream': ('twitch.tv/', 'TwitchStream'),
 114         'twitter': ('twitter.com/i/cards/tfw/v1/', 'TwitterCard'),
 115         'ustream-channel': ('ustream.tv/embed/', 'Ustream'),
 116         'vimeo': ('vimeo.com/', 'Vimeo'),
 117         'vine': ('vine.co/v/', 'Vine'),
 118         'youtube-list': ('youtube.com/embed/%s?list=%s', 'YoutubePlaylist'),
 119         'youtube-video': ('youtube.com/embed/', 'Youtube'),
 120     }
 121
 122     @staticmethod
 123     def _extract_urls(webpage, url):
 124         return [urljoin(url, unescapeHTML(mobj.group('url'))) for mobj in re.finditer(
 125             r'(?x)<iframe[^>]+?src=(?P<q>["\'])(?P<url>(?:(?:https?:)?//%s)?%s(?:(?!\1).)+)\1' % (KinjaEmbedIE._DOMAIN_REGEX, KinjaEmbedIE._COMMON_REGEX),
 126             webpage)]
 127
 128     def _real_extract(self, url):
 129         video_type, video_id = self._match_valid_url(url).groups()
 130
 131         provider = self._PROVIDER_MAP.get(video_type)
 132         if provider:
 133             video_id = compat_urllib_parse_unquote(video_id)
 134             if video_type == 'tumblr-post':
 135                 video_id, blog = video_id.split('-', 1)
 136                 result_url = provider[0] % (blog, video_id)
 137             elif video_type == 'youtube-list':
 138                 video_id, playlist_id = video_id.split('/')
 139                 result_url = provider[0] % (video_id, playlist_id)
 140             else:
 141                 if video_type == 'ooyala':
 142                     video_id = video_id.split('/')[0]
 143                 result_url = provider[0] + video_id
 144             return self.url_result('http://' + result_url, provider[1])
 145
 146         if video_type == 'kinjavideo':
 147             data = self._download_json(
 148                 'https://kinja.com/api/core/video/views/videoById',
 149                 video_id, query={'videoId': video_id})['data']
 150             title = data['title']
 151
 152             formats = []
 153             for k in ('signedPlaylist', 'streaming'):
 154                 m3u8_url = data.get(k + 'Url')
 155                 if m3u8_url:
 156                     formats.extend(self._extract_m3u8_formats(
 157                         m3u8_url, video_id, 'mp4', 'm3u8_native',
 158                         m3u8_id='hls', fatal=False))
 159             self._sort_formats(formats)
 160
 161             thumbnail = None
 162             poster = data.get('poster') or {}
 163             poster_id = poster.get('id')
 164             if poster_id:
 165                 thumbnail = 'https://i.kinja-img.com/gawker-media/image/upload/%s.%s' % (poster_id, poster.get('format') or 'jpg')
 166
 167             return {
 168                 'id': video_id,
 169                 'title': title,
 170                 'description': strip_or_none(data.get('description')),
 171                 'formats': formats,
 172                 'tags': data.get('tags'),
 173                 'timestamp': int_or_none(try_get(
 174                     data, lambda x: x['postInfo']['publishTimeMillis']), 1000),
 175                 'thumbnail': thumbnail,
 176                 'uploader': data.get('network'),
 177             }
 178         else:
 179             video_data = self._download_json(
 180                 'https://api.vmh.univision.com/metadata/v1/content/' + video_id,
 181                 video_id)['videoMetadata']
 182             iptc = video_data['photoVideoMetadataIPTC']
 183             title = iptc['title']['en']
 184             fmg = video_data.get('photoVideoMetadata_fmg') or {}
 185             tvss_domain = fmg.get('tvssDomain') or 'https://auth.univision.com'
 186             data = self._download_json(
 187                 tvss_domain + '/api/v3/video-auth/url-signature-tokens',
 188                 video_id, query={'mcpids': video_id})['data'][0]
 189             formats = []
 190
 191             rendition_url = data.get('renditionUrl')
 192             if rendition_url:
 193                 formats = self._extract_m3u8_formats(
 194                     rendition_url, video_id, 'mp4',
 195                     'm3u8_native', m3u8_id='hls', fatal=False)
 196
 197             fallback_rendition_url = data.get('fallbackRenditionUrl')
 198             if fallback_rendition_url:
 199                 formats.append({
 200                     'format_id': 'fallback',
 201                     'tbr': int_or_none(self._search_regex(
 202                         r'_(\d+)\.mp4', fallback_rendition_url,
 203                         'bitrate', default=None)),
 204                     'url': fallback_rendition_url,
 205                 })
 206
 207             self._sort_formats(formats)
 208
 209             return {
 210                 'id': video_id,
 211                 'title': title,
 212                 'thumbnail': try_get(iptc, lambda x: x['cloudinaryLink']['link'], compat_str),
 213                 'uploader': fmg.get('network'),
 214                 'duration': int_or_none(iptc.get('fileDuration')),
 215                 'formats': formats,
 216                 'description': try_get(iptc, lambda x: x['description']['en'], compat_str),
 217                 'timestamp': parse_iso8601(iptc.get('dateReleased')),
 218             }