yt_dlp/extractor/aenetworks.py

   1 from .theplatform import ThePlatformIE
   2 from ..utils import (
   3     ExtractorError,
   4     GeoRestrictedError,
   5     int_or_none,
   6     update_url_query,
   7     urlencode_postdata,
   8 )
   9
  10
  11 class AENetworksBaseIE(ThePlatformIE):
  12     _BASE_URL_REGEX = r'''(?x)https?://
  13         (?:(?:www|play|watch)\.)?
  14         (?P<domain>
  15             (?:history(?:vault)?|aetv|mylifetime|lifetimemovieclub)\.com|
  16             fyi\.tv
  17         )/'''
  18     _THEPLATFORM_KEY = '43jXaGRQud'
  19     _THEPLATFORM_SECRET = 'S10BPXHMlb'
  20     _DOMAIN_MAP = {
  21         'history.com': ('HISTORY', 'history'),
  22         'aetv.com': ('AETV', 'aetv'),
  23         'mylifetime.com': ('LIFETIME', 'lifetime'),
  24         'lifetimemovieclub.com': ('LIFETIMEMOVIECLUB', 'lmc'),
  25         'fyi.tv': ('FYI', 'fyi'),
  26         'historyvault.com': (None, 'historyvault'),
  27         'biography.com': (None, 'biography'),
  28     }
  29
  30     def _extract_aen_smil(self, smil_url, video_id, auth=None):
  31         query = {
  32             'mbr': 'true',
  33             'formats': 'M3U+none,MPEG-DASH+none,MPEG4,MP3',
  34         }
  35         if auth:
  36             query['auth'] = auth
  37         TP_SMIL_QUERY = [{
  38             'assetTypes': 'high_video_ak',
  39             'switch': 'hls_high_ak',
  40         }, {
  41             'assetTypes': 'high_video_s3',
  42         }, {
  43             'assetTypes': 'high_video_s3',
  44             'switch': 'hls_high_fastly',
  45         }]
  46         formats = []
  47         subtitles = {}
  48         last_e = None
  49         for q in TP_SMIL_QUERY:
  50             q.update(query)
  51             m_url = update_url_query(smil_url, q)
  52             m_url = self._sign_url(m_url, self._THEPLATFORM_KEY, self._THEPLATFORM_SECRET)
  53             try:
  54                 tp_formats, tp_subtitles = self._extract_theplatform_smil(
  55                     m_url, video_id, 'Downloading %s SMIL data' % (q.get('switch') or q['assetTypes']))
  56             except ExtractorError as e:
  57                 if isinstance(e, GeoRestrictedError):
  58                     raise
  59                 last_e = e
  60                 continue
  61             formats.extend(tp_formats)
  62             subtitles = self._merge_subtitles(subtitles, tp_subtitles)
  63         if last_e and not formats:
  64             raise last_e
  65         self._sort_formats(formats)
  66         return {
  67             'id': video_id,
  68             'formats': formats,
  69             'subtitles': subtitles,
  70         }
  71
  72     def _extract_aetn_info(self, domain, filter_key, filter_value, url):
  73         requestor_id, brand = self._DOMAIN_MAP[domain]
  74         result = self._download_json(
  75             'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
  76             filter_value, query={'filter[%s]' % filter_key: filter_value})['results'][0]
  77         title = result['title']
  78         video_id = result['id']
  79         media_url = result['publicUrl']
  80         theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
  81             r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
  82         info = self._parse_theplatform_metadata(theplatform_metadata)
  83         auth = None
  84         if theplatform_metadata.get('AETN$isBehindWall'):
  85             resource = self._get_mvpd_resource(
  86                 requestor_id, theplatform_metadata['title'],
  87                 theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
  88                 theplatform_metadata['ratings'][0]['rating'])
  89             auth = self._extract_mvpd_auth(
  90                 url, video_id, requestor_id, resource)
  91         info.update(self._extract_aen_smil(media_url, video_id, auth))
  92         info.update({
  93             'title': title,
  94             'series': result.get('seriesName'),
  95             'season_number': int_or_none(result.get('tvSeasonNumber')),
  96             'episode_number': int_or_none(result.get('tvSeasonEpisodeNumber')),
  97         })
  98         return info
  99
 100
 101 class AENetworksIE(AENetworksBaseIE):
 102     IE_NAME = 'aenetworks'
 103     IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault'
 104     _VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'''(?P<id>
 105         shows/[^/]+/season-\d+/episode-\d+|
 106         (?:
 107             (?:movie|special)s/[^/]+|
 108             (?:shows/[^/]+/)?videos
 109         )/[^/?#&]+
 110     )'''
 111     _TESTS = [{
 112         'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
 113         'info_dict': {
 114             'id': '22253814',
 115             'ext': 'mp4',
 116             'title': 'Winter is Coming',
 117             'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
 118             'timestamp': 1338306241,
 119             'upload_date': '20120529',
 120             'uploader': 'AENE-NEW',
 121         },
 122         'params': {
 123             # m3u8 download
 124             'skip_download': True,
 125         },
 126         'add_ie': ['ThePlatform'],
 127         'skip': 'This video is only available for users of participating TV providers.',
 128     }, {
 129         'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
 130         'info_dict': {
 131             'id': '600587331957',
 132             'ext': 'mp4',
 133             'title': 'Inlawful Entry',
 134             'description': 'md5:57c12115a2b384d883fe64ca50529e08',
 135             'timestamp': 1452634428,
 136             'upload_date': '20160112',
 137             'uploader': 'AENE-NEW',
 138         },
 139         'params': {
 140             # m3u8 download
 141             'skip_download': True,
 142         },
 143         'add_ie': ['ThePlatform'],
 144     }, {
 145         'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
 146         'only_matching': True
 147     }, {
 148         'url': 'http://www.mylifetime.com/shows/project-runway-junior/season-1/episode-6',
 149         'only_matching': True
 150     }, {
 151         'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
 152         'only_matching': True
 153     }, {
 154         'url': 'https://watch.lifetimemovieclub.com/movies/10-year-reunion/full-movie',
 155         'only_matching': True
 156     }, {
 157         'url': 'http://www.history.com/specials/sniper-into-the-kill-zone/full-special',
 158         'only_matching': True
 159     }, {
 160         'url': 'https://www.aetv.com/specials/hunting-jonbenets-killer-the-untold-story/preview-hunting-jonbenets-killer-the-untold-story',
 161         'only_matching': True
 162     }, {
 163         'url': 'http://www.history.com/videos/history-of-valentines-day',
 164         'only_matching': True
 165     }, {
 166         'url': 'https://play.aetv.com/shows/duck-dynasty/videos/best-of-duck-dynasty-getting-quack-in-shape',
 167         'only_matching': True
 168     }]
 169
 170     def _real_extract(self, url):
 171         domain, canonical = self._match_valid_url(url).groups()
 172         return self._extract_aetn_info(domain, 'canonical', '/' + canonical, url)
 173
 174
 175 class AENetworksListBaseIE(AENetworksBaseIE):
 176     def _call_api(self, resource, slug, brand, fields):
 177         return self._download_json(
 178             'https://yoga.appsvcs.aetnd.com/graphql',
 179             slug, query={'brand': brand}, data=urlencode_postdata({
 180                 'query': '''{
 181   %s(slug: "%s") {
 182     %s
 183   }
 184 }''' % (resource, slug, fields),
 185             }))['data'][resource]
 186
 187     def _real_extract(self, url):
 188         domain, slug = self._match_valid_url(url).groups()
 189         _, brand = self._DOMAIN_MAP[domain]
 190         playlist = self._call_api(self._RESOURCE, slug, brand, self._FIELDS)
 191         base_url = 'http://watch.%s' % domain
 192
 193         entries = []
 194         for item in (playlist.get(self._ITEMS_KEY) or []):
 195             doc = self._get_doc(item)
 196             canonical = doc.get('canonical')
 197             if not canonical:
 198                 continue
 199             entries.append(self.url_result(
 200                 base_url + canonical, AENetworksIE.ie_key(), doc.get('id')))
 201
 202         description = None
 203         if self._PLAYLIST_DESCRIPTION_KEY:
 204             description = playlist.get(self._PLAYLIST_DESCRIPTION_KEY)
 205
 206         return self.playlist_result(
 207             entries, playlist.get('id'),
 208             playlist.get(self._PLAYLIST_TITLE_KEY), description)
 209
 210
 211 class AENetworksCollectionIE(AENetworksListBaseIE):
 212     IE_NAME = 'aenetworks:collection'
 213     _VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'(?:[^/]+/)*(?:list|collections)/(?P<id>[^/?#&]+)/?(?:[?#&]|$)'
 214     _TESTS = [{
 215         'url': 'https://watch.historyvault.com/list/america-the-story-of-us',
 216         'info_dict': {
 217             'id': '282',
 218             'title': 'America The Story of Us',
 219         },
 220         'playlist_mincount': 12,
 221     }, {
 222         'url': 'https://watch.historyvault.com/shows/america-the-story-of-us-2/season-1/list/america-the-story-of-us',
 223         'only_matching': True
 224     }, {
 225         'url': 'https://www.historyvault.com/collections/mysteryquest',
 226         'only_matching': True
 227     }]
 228     _RESOURCE = 'list'
 229     _ITEMS_KEY = 'items'
 230     _PLAYLIST_TITLE_KEY = 'display_title'
 231     _PLAYLIST_DESCRIPTION_KEY = None
 232     _FIELDS = '''id
 233     display_title
 234     items {
 235       ... on ListVideoItem {
 236         doc {
 237           canonical
 238           id
 239         }
 240       }
 241     }'''
 242
 243     def _get_doc(self, item):
 244         return item.get('doc') or {}
 245
 246
 247 class AENetworksShowIE(AENetworksListBaseIE):
 248     IE_NAME = 'aenetworks:show'
 249     _VALID_URL = AENetworksBaseIE._BASE_URL_REGEX + r'shows/(?P<id>[^/?#&]+)/?(?:[?#&]|$)'
 250     _TESTS = [{
 251         'url': 'http://www.history.com/shows/ancient-aliens',
 252         'info_dict': {
 253             'id': 'SERIES1574',
 254             'title': 'Ancient Aliens',
 255             'description': 'md5:3f6d74daf2672ff3ae29ed732e37ea7f',
 256         },
 257         'playlist_mincount': 150,
 258     }]
 259     _RESOURCE = 'series'
 260     _ITEMS_KEY = 'episodes'
 261     _PLAYLIST_TITLE_KEY = 'title'
 262     _PLAYLIST_DESCRIPTION_KEY = 'description'
 263     _FIELDS = '''description
 264     id
 265     title
 266     episodes {
 267       canonical
 268       id
 269     }'''
 270
 271     def _get_doc(self, item):
 272         return item
 273
 274
 275 class HistoryTopicIE(AENetworksBaseIE):
 276     IE_NAME = 'history:topic'
 277     IE_DESC = 'History.com Topic'
 278     _VALID_URL = r'https?://(?:www\.)?history\.com/topics/[^/]+/(?P<id>[\w+-]+?)-video'
 279     _TESTS = [{
 280         'url': 'https://www.history.com/topics/valentines-day/history-of-valentines-day-video',
 281         'info_dict': {
 282             'id': '40700995724',
 283             'ext': 'mp4',
 284             'title': "History of Valentine’s Day",
 285             'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
 286             'timestamp': 1375819729,
 287             'upload_date': '20130806',
 288             'uploader': 'AENE-NEW',
 289         },
 290         'params': {
 291             # m3u8 download
 292             'skip_download': True,
 293         },
 294         'add_ie': ['ThePlatform'],
 295     }]
 296
 297     def _real_extract(self, url):
 298         display_id = self._match_id(url)
 299         return self.url_result(
 300             'http://www.history.com/videos/' + display_id,
 301             AENetworksIE.ie_key())
 302
 303
 304 class HistoryPlayerIE(AENetworksBaseIE):
 305     IE_NAME = 'history:player'
 306     _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|biography)\.com)/player/(?P<id>\d+)'
 307     _TESTS = []
 308
 309     def _real_extract(self, url):
 310         domain, video_id = self._match_valid_url(url).groups()
 311         return self._extract_aetn_info(domain, 'id', video_id, url)
 312
 313
 314 class BiographyIE(AENetworksBaseIE):
 315     _VALID_URL = r'https?://(?:www\.)?biography\.com/video/(?P<id>[^/?#&]+)'
 316     _TESTS = [{
 317         'url': 'https://www.biography.com/video/vincent-van-gogh-full-episode-2075049808',
 318         'info_dict': {
 319             'id': '30322987',
 320             'ext': 'mp4',
 321             'title': 'Vincent Van Gogh - Full Episode',
 322             'description': 'A full biography about the most influential 20th century painter, Vincent Van Gogh.',
 323             'timestamp': 1311970571,
 324             'upload_date': '20110729',
 325             'uploader': 'AENE-NEW',
 326         },
 327         'params': {
 328             # m3u8 download
 329             'skip_download': True,
 330         },
 331         'add_ie': ['ThePlatform'],
 332     }]
 333
 334     def _real_extract(self, url):
 335         display_id = self._match_id(url)
 336         webpage = self._download_webpage(url, display_id)
 337         player_url = self._search_regex(
 338             r'<phoenix-iframe[^>]+src="(%s)' % HistoryPlayerIE._VALID_URL,
 339             webpage, 'player URL')
 340         return self.url_result(player_url, HistoryPlayerIE.ie_key())