yt_dlp/extractor/europa.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     int_or_none,
   4     orderedSet,
   5     parse_duration,
   6     parse_iso8601,
   7     parse_qs,
   8     qualities,
   9     traverse_obj,
  10     unified_strdate,
  11     xpath_text
  12 )
  13
  14
  15 class EuropaIE(InfoExtractor):
  16     _WORKING = False
  17     _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9-]+)'
  18     _TESTS = [{
  19         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
  20         'md5': '574f080699ddd1e19a675b0ddf010371',
  21         'info_dict': {
  22             'id': 'I107758',
  23             'ext': 'mp4',
  24             'title': 'TRADE - Wikileaks on TTIP',
  25             'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
  26             'thumbnail': r're:^https?://.*\.jpg$',
  27             'upload_date': '20150811',
  28             'duration': 34,
  29             'view_count': int,
  30             'formats': 'mincount:3',
  31         }
  32     }, {
  33         'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
  34         'only_matching': True,
  35     }, {
  36         'url': 'http://ec.europa.eu/avservices/audio/audioDetails.cfm?ref=I-109295&sitelang=en',
  37         'only_matching': True,
  38     }]
  39
  40     def _real_extract(self, url):
  41         video_id = self._match_id(url)
  42
  43         playlist = self._download_xml(
  44             'http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=%s' % video_id, video_id)
  45
  46         def get_item(type_, preference):
  47             items = {}
  48             for item in playlist.findall('./info/%s/item' % type_):
  49                 lang, label = xpath_text(item, 'lg', default=None), xpath_text(item, 'label', default=None)
  50                 if lang and label:
  51                     items[lang] = label.strip()
  52             for p in preference:
  53                 if items.get(p):
  54                     return items[p]
  55
  56         query = parse_qs(url)
  57         preferred_lang = query.get('sitelang', ('en', ))[0]
  58
  59         preferred_langs = orderedSet((preferred_lang, 'en', 'int'))
  60
  61         title = get_item('title', preferred_langs) or video_id
  62         description = get_item('description', preferred_langs)
  63         thumbnail = xpath_text(playlist, './info/thumburl', 'thumbnail')
  64         upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
  65         duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
  66         view_count = int_or_none(xpath_text(playlist, './info/views', 'views'))
  67
  68         language_preference = qualities(preferred_langs[::-1])
  69
  70         formats = []
  71         for file_ in playlist.findall('./files/file'):
  72             video_url = xpath_text(file_, './url')
  73             if not video_url:
  74                 continue
  75             lang = xpath_text(file_, './lg')
  76             formats.append({
  77                 'url': video_url,
  78                 'format_id': lang,
  79                 'format_note': xpath_text(file_, './lglabel'),
  80                 'language_preference': language_preference(lang)
  81             })
  82
  83         return {
  84             'id': video_id,
  85             'title': title,
  86             'description': description,
  87             'thumbnail': thumbnail,
  88             'upload_date': upload_date,
  89             'duration': duration,
  90             'view_count': view_count,
  91             'formats': formats
  92         }
  93
  94
  95 class EuroParlWebstreamIE(InfoExtractor):
  96     _VALID_URL = r'''(?x)
  97         https?://multimedia\.europarl\.europa\.eu/[^/#?]+/
  98         (?:(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
  99     '''
 100     _TESTS = [{
 101         'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/plenary-session_20220914-0900-PLENARY',
 102         'info_dict': {
 103             'id': '62388b15-d85b-4add-99aa-ba12ccf64f0d',
 104             'ext': 'mp4',
 105             'title': 'Plenary session',
 106             'release_timestamp': 1663139069,
 107             'release_date': '20220914',
 108         },
 109         'params': {
 110             'skip_download': True,
 111         }
 112     }, {
 113         # live webstream
 114         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/euroscola_20221115-1000-SPECIAL-EUROSCOLA',
 115         'info_dict': {
 116             'ext': 'mp4',
 117             'id': '510eda7f-ba72-161b-7ee7-0e836cd2e715',
 118             'release_timestamp': 1668502800,
 119             'title': 'Euroscola 2022-11-15 19:21',
 120             'release_date': '20221115',
 121             'live_status': 'is_live',
 122         },
 123         'skip': 'not live anymore'
 124     }, {
 125         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-culture-and-education_20230301-1130-COMMITTEE-CULT',
 126         'info_dict': {
 127             'id': '7355662c-8eac-445e-4bb9-08db14b0ddd7',
 128             'ext': 'mp4',
 129             'release_date': '20230301',
 130             'title': 'Committee on Culture and Education',
 131             'release_timestamp': 1677666641,
 132         }
 133     }, {
 134         # live stream
 135         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-environment-public-health-and-food-safety_20230524-0900-COMMITTEE-ENVI',
 136         'info_dict': {
 137             'id': 'e4255f56-10aa-4b3c-6530-08db56d5b0d9',
 138             'ext': 'mp4',
 139             'release_date': '20230524',
 140             'title': r're:Committee on Environment, Public Health and Food Safety \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}',
 141             'release_timestamp': 1684911541,
 142             'live_status': 'is_live',
 143         },
 144         'skip': 'Not live anymore'
 145     }]
 146
 147     def _real_extract(self, url):
 148         display_id = self._match_id(url)
 149         webpage = self._download_webpage(url, display_id)
 150
 151         webpage_nextjs = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
 152
 153         json_info = self._download_json(
 154             'https://acs-api.europarl.connectedviews.eu/api/FullMeeting', display_id,
 155             query={
 156                 'api-version': 1.0,
 157                 'tenantId': 'bae646ca-1fc8-4363-80ba-2c04f06b4968',
 158                 'externalReference': display_id
 159             })
 160
 161         formats, subtitles = [], {}
 162         for hls_url in traverse_obj(json_info, ((('meetingVideo'), ('meetingVideos', ...)), 'hlsUrl')):
 163             fmt, subs = self._extract_m3u8_formats_and_subtitles(hls_url, display_id)
 164             formats.extend(fmt)
 165             self._merge_subtitles(subs, target=subtitles)
 166
 167         return {
 168             'id': json_info['id'],
 169             'title': traverse_obj(webpage_nextjs, (('mediaItem', 'title'), ('title', )), get_all=False),
 170             'formats': formats,
 171             'subtitles': subtitles,
 172             'release_timestamp': parse_iso8601(json_info.get('startDateTime')),
 173             'is_live': traverse_obj(webpage_nextjs, ('mediaItem', 'mediaSubType')) == 'Live'
 174         }