yt_dlp/extractor/arte.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     compat_str,
   9 )
  10 from ..utils import (
  11     ExtractorError,
  12     int_or_none,
  13     parse_qs,
  14     qualities,
  15     strip_or_none,
  16     try_get,
  17     unified_strdate,
  18     url_or_none,
  19 )
  20
  21
  22 class ArteTVBaseIE(InfoExtractor):
  23     _ARTE_LANGUAGES = 'fr|de|en|es|it|pl'
  24     _API_BASE = 'https://api.arte.tv/api/player/v1'
  25
  26
  27 class ArteTVIE(ArteTVBaseIE):
  28     _VALID_URL = r'''(?x)
  29                     https?://
  30                         (?:
  31                             (?:www\.)?arte\.tv/(?P<lang>%(langs)s)/videos|
  32                             api\.arte\.tv/api/player/v\d+/config/(?P<lang_2>%(langs)s)
  33                         )
  34                         /(?P<id>\d{6}-\d{3}-[AF])
  35                     ''' % {'langs': ArteTVBaseIE._ARTE_LANGUAGES}
  36     _TESTS = [{
  37         'url': 'https://www.arte.tv/en/videos/088501-000-A/mexico-stealing-petrol-to-survive/',
  38         'info_dict': {
  39             'id': '088501-000-A',
  40             'ext': 'mp4',
  41             'title': 'Mexico: Stealing Petrol to Survive',
  42             'upload_date': '20190628',
  43         },
  44     }, {
  45         'url': 'https://www.arte.tv/pl/videos/100103-000-A/usa-dyskryminacja-na-porodowce/',
  46         'only_matching': True,
  47     }, {
  48         'url': 'https://api.arte.tv/api/player/v2/config/de/100605-013-A',
  49         'only_matching': True,
  50     }]
  51
  52     def _real_extract(self, url):
  53         mobj = self._match_valid_url(url)
  54         video_id = mobj.group('id')
  55         lang = mobj.group('lang') or mobj.group('lang_2')
  56
  57         info = self._download_json(
  58             '%s/config/%s/%s' % (self._API_BASE, lang, video_id), video_id)
  59         player_info = info['videoJsonPlayer']
  60
  61         vsr = try_get(player_info, lambda x: x['VSR'], dict)
  62         if not vsr:
  63             error = None
  64             if try_get(player_info, lambda x: x['custom_msg']['type']) == 'error':
  65                 error = try_get(
  66                     player_info, lambda x: x['custom_msg']['msg'], compat_str)
  67             if not error:
  68                 error = 'Video %s is not available' % player_info.get('VID') or video_id
  69             raise ExtractorError(error, expected=True)
  70
  71         upload_date_str = player_info.get('shootingDate')
  72         if not upload_date_str:
  73             upload_date_str = (player_info.get('VRA') or player_info.get('VDA') or '').split(' ')[0]
  74
  75         title = (player_info.get('VTI') or player_info['VID']).strip()
  76         subtitle = player_info.get('VSU', '').strip()
  77         if subtitle:
  78             title += ' - %s' % subtitle
  79
  80         qfunc = qualities(['MQ', 'HQ', 'EQ', 'SQ'])
  81
  82         LANGS = {
  83             'fr': 'F',
  84             'de': 'A',
  85             'en': 'E[ANG]',
  86             'es': 'E[ESP]',
  87             'it': 'E[ITA]',
  88             'pl': 'E[POL]',
  89         }
  90
  91         langcode = LANGS.get(lang, lang)
  92
  93         formats = []
  94         for format_id, format_dict in vsr.items():
  95             f = dict(format_dict)
  96             format_url = url_or_none(f.get('url'))
  97             streamer = f.get('streamer')
  98             if not format_url and not streamer:
  99                 continue
 100             versionCode = f.get('versionCode')
 101             l = re.escape(langcode)
 102
 103             # Language preference from most to least priority
 104             # Reference: section 6.8 of
 105             # https://www.arte.tv/sites/en/corporate/files/complete-technical-guidelines-arte-geie-v1-07-1.pdf
 106             PREFERENCES = (
 107                 # original version in requested language, without subtitles
 108                 r'VO{0}$'.format(l),
 109                 # original version in requested language, with partial subtitles in requested language
 110                 r'VO{0}-ST{0}$'.format(l),
 111                 # original version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
 112                 r'VO{0}-STM{0}$'.format(l),
 113                 # non-original (dubbed) version in requested language, without subtitles
 114                 r'V{0}$'.format(l),
 115                 # non-original (dubbed) version in requested language, with subtitles partial subtitles in requested language
 116                 r'V{0}-ST{0}$'.format(l),
 117                 # non-original (dubbed) version in requested language, with subtitles for the deaf and hard-of-hearing in requested language
 118                 r'V{0}-STM{0}$'.format(l),
 119                 # original version in requested language, with partial subtitles in different language
 120                 r'VO{0}-ST(?!{0}).+?$'.format(l),
 121                 # original version in requested language, with subtitles for the deaf and hard-of-hearing in different language
 122                 r'VO{0}-STM(?!{0}).+?$'.format(l),
 123                 # original version in different language, with partial subtitles in requested language
 124                 r'VO(?:(?!{0}).+?)?-ST{0}$'.format(l),
 125                 # original version in different language, with subtitles for the deaf and hard-of-hearing in requested language
 126                 r'VO(?:(?!{0}).+?)?-STM{0}$'.format(l),
 127                 # original version in different language, without subtitles
 128                 r'VO(?:(?!{0}))?$'.format(l),
 129                 # original version in different language, with partial subtitles in different language
 130                 r'VO(?:(?!{0}).+?)?-ST(?!{0}).+?$'.format(l),
 131                 # original version in different language, with subtitles for the deaf and hard-of-hearing in different language
 132                 r'VO(?:(?!{0}).+?)?-STM(?!{0}).+?$'.format(l),
 133             )
 134
 135             for pref, p in enumerate(PREFERENCES):
 136                 if re.match(p, versionCode):
 137                     lang_pref = len(PREFERENCES) - pref
 138                     break
 139             else:
 140                 lang_pref = -1
 141             format_note = '%s, %s' % (f.get('versionCode'), f.get('versionLibelle'))
 142
 143             media_type = f.get('mediaType')
 144             if media_type == 'hls':
 145                 m3u8_formats = self._extract_m3u8_formats(
 146                     format_url, video_id, 'mp4', entry_protocol='m3u8_native',
 147                     m3u8_id=format_id, fatal=False)
 148                 for m3u8_format in m3u8_formats:
 149                     m3u8_format.update({
 150                         'language_preference': lang_pref,
 151                         'format_note': format_note,
 152                     })
 153                 formats.extend(m3u8_formats)
 154                 continue
 155
 156             format = {
 157                 'format_id': format_id,
 158                 'language_preference': lang_pref,
 159                 'format_note': format_note,
 160                 'width': int_or_none(f.get('width')),
 161                 'height': int_or_none(f.get('height')),
 162                 'tbr': int_or_none(f.get('bitrate')),
 163                 'quality': qfunc(f.get('quality')),
 164             }
 165
 166             if media_type == 'rtmp':
 167                 format['url'] = f['streamer']
 168                 format['play_path'] = 'mp4:' + f['url']
 169                 format['ext'] = 'flv'
 170             else:
 171                 format['url'] = f['url']
 172
 173             formats.append(format)
 174
 175         # For this extractor, quality only represents the relative quality
 176         # with respect to other formats with the same resolution
 177         self._sort_formats(formats, ('res', 'quality'))
 178
 179         return {
 180             'id': player_info.get('VID') or video_id,
 181             'title': title,
 182             'description': player_info.get('VDE') or player_info.get('V7T'),
 183             'upload_date': unified_strdate(upload_date_str),
 184             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
 185             'formats': formats,
 186         }
 187
 188
 189 class ArteTVEmbedIE(InfoExtractor):
 190     _VALID_URL = r'https?://(?:www\.)?arte\.tv/player/v\d+/index\.php\?.*?\bjson_url=.+'
 191     _TESTS = [{
 192         'url': 'https://www.arte.tv/player/v5/index.php?json_url=https%3A%2F%2Fapi.arte.tv%2Fapi%2Fplayer%2Fv2%2Fconfig%2Fde%2F100605-013-A&lang=de&autoplay=true&mute=0100605-013-A',
 193         'info_dict': {
 194             'id': '100605-013-A',
 195             'ext': 'mp4',
 196             'title': 'United we Stream November Lockdown Edition #13',
 197             'description': 'md5:be40b667f45189632b78c1425c7c2ce1',
 198             'upload_date': '20201116',
 199         },
 200     }, {
 201         'url': 'https://www.arte.tv/player/v3/index.php?json_url=https://api.arte.tv/api/player/v2/config/de/100605-013-A',
 202         'only_matching': True,
 203     }]
 204
 205     @staticmethod
 206     def _extract_urls(webpage):
 207         return [url for _, url in re.findall(
 208             r'<(?:iframe|script)[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?arte\.tv/player/v\d+/index\.php\?.*?\bjson_url=.+?)\1',
 209             webpage)]
 210
 211     def _real_extract(self, url):
 212         qs = parse_qs(url)
 213         json_url = qs['json_url'][0]
 214         video_id = ArteTVIE._match_id(json_url)
 215         return self.url_result(
 216             json_url, ie=ArteTVIE.ie_key(), video_id=video_id)
 217
 218
 219 class ArteTVPlaylistIE(ArteTVBaseIE):
 220     _VALID_URL = r'https?://(?:www\.)?arte\.tv/(?P<lang>%s)/videos/(?P<id>RC-\d{6})' % ArteTVBaseIE._ARTE_LANGUAGES
 221     _TESTS = [{
 222         'url': 'https://www.arte.tv/en/videos/RC-016954/earn-a-living/',
 223         'info_dict': {
 224             'id': 'RC-016954',
 225             'title': 'Earn a Living',
 226             'description': 'md5:d322c55011514b3a7241f7fb80d494c2',
 227         },
 228         'playlist_mincount': 6,
 229     }, {
 230         'url': 'https://www.arte.tv/pl/videos/RC-014123/arte-reportage/',
 231         'only_matching': True,
 232     }]
 233
 234     def _real_extract(self, url):
 235         lang, playlist_id = self._match_valid_url(url).groups()
 236         collection = self._download_json(
 237             '%s/collectionData/%s/%s?source=videos'
 238             % (self._API_BASE, lang, playlist_id), playlist_id)
 239         entries = []
 240         for video in collection['videos']:
 241             if not isinstance(video, dict):
 242                 continue
 243             video_url = url_or_none(video.get('url')) or url_or_none(video.get('jsonUrl'))
 244             if not video_url:
 245                 continue
 246             video_id = video.get('programId')
 247             entries.append({
 248                 '_type': 'url_transparent',
 249                 'url': video_url,
 250                 'id': video_id,
 251                 'title': video.get('title'),
 252                 'alt_title': video.get('subtitle'),
 253                 'thumbnail': url_or_none(try_get(video, lambda x: x['mainImage']['url'], compat_str)),
 254                 'duration': int_or_none(video.get('durationSeconds')),
 255                 'view_count': int_or_none(video.get('views')),
 256                 'ie_key': ArteTVIE.ie_key(),
 257             })
 258         title = collection.get('title')
 259         description = collection.get('shortDescription') or collection.get('teaserText')
 260         return self.playlist_result(entries, playlist_id, title, description)
 261
 262
 263 class ArteTVCategoryIE(ArteTVBaseIE):
 264     _VALID_URL = r'https?://(?:www\.)?arte\.tv/(?P<lang>%s)/videos/(?P<id>[\w-]+(?:/[\w-]+)*)/?\s*$' % ArteTVBaseIE._ARTE_LANGUAGES
 265     _TESTS = [{
 266         'url': 'https://www.arte.tv/en/videos/politics-and-society/',
 267         'info_dict': {
 268             'id': 'politics-and-society',
 269             'title': 'Politics and society',
 270             'description': 'Investigative documentary series, geopolitical analysis, and international commentary',
 271         },
 272         'playlist_mincount': 13,
 273     },
 274     ]
 275
 276     @classmethod
 277     def suitable(cls, url):
 278         return (
 279             not any(ie.suitable(url) for ie in (ArteTVIE, ArteTVPlaylistIE, ))
 280             and super(ArteTVCategoryIE, cls).suitable(url))
 281
 282     def _real_extract(self, url):
 283         lang, playlist_id = self._match_valid_url(url).groups()
 284         webpage = self._download_webpage(url, playlist_id)
 285
 286         items = []
 287         for video in re.finditer(
 288                 r'<a\b[^>]*?href\s*=\s*(?P<q>"|\'|\b)(?P<url>https?://www\.arte\.tv/%s/videos/[\w/-]+)(?P=q)' % lang,
 289                 webpage):
 290             video = video.group('url')
 291             if video == url:
 292                 continue
 293             if any(ie.suitable(video) for ie in (ArteTVIE, ArteTVPlaylistIE, )):
 294                 items.append(video)
 295
 296         title = (self._og_search_title(webpage, default=None)
 297                  or self._html_search_regex(r'<title\b[^>]*>([^<]+)</title>', default=None))
 298         title = strip_or_none(title.rsplit('|', 1)[0]) or self._generic_title(url)
 299
 300         return self.playlist_from_matches(items, playlist_id=playlist_id, playlist_title=title,
 301                                           description=self._og_search_description(webpage, default=None))