yt_dlp/extractor/europa.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     int_or_none,
   4     orderedSet,
   5     parse_duration,
   6     parse_iso8601,
   7     parse_qs,
   8     qualities,
   9     traverse_obj,
  10     unified_strdate,
  11     xpath_text
  12 )
  13
  14
  15 class EuropaIE(InfoExtractor):
  16     _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9-]+)'
  17     _TESTS = [{
  18         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
  19         'md5': '574f080699ddd1e19a675b0ddf010371',
  20         'info_dict': {
  21             'id': 'I107758',
  22             'ext': 'mp4',
  23             'title': 'TRADE - Wikileaks on TTIP',
  24             'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
  25             'thumbnail': r're:^https?://.*\.jpg$',
  26             'upload_date': '20150811',
  27             'duration': 34,
  28             'view_count': int,
  29             'formats': 'mincount:3',
  30         }
  31     }, {
  32         'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
  33         'only_matching': True,
  34     }, {
  35         'url': 'http://ec.europa.eu/avservices/audio/audioDetails.cfm?ref=I-109295&sitelang=en',
  36         'only_matching': True,
  37     }]
  38
  39     def _real_extract(self, url):
  40         video_id = self._match_id(url)
  41
  42         playlist = self._download_xml(
  43             'http://ec.europa.eu/avservices/video/player/playlist.cfm?ID=%s' % video_id, video_id)
  44
  45         def get_item(type_, preference):
  46             items = {}
  47             for item in playlist.findall('./info/%s/item' % type_):
  48                 lang, label = xpath_text(item, 'lg', default=None), xpath_text(item, 'label', default=None)
  49                 if lang and label:
  50                     items[lang] = label.strip()
  51             for p in preference:
  52                 if items.get(p):
  53                     return items[p]
  54
  55         query = parse_qs(url)
  56         preferred_lang = query.get('sitelang', ('en', ))[0]
  57
  58         preferred_langs = orderedSet((preferred_lang, 'en', 'int'))
  59
  60         title = get_item('title', preferred_langs) or video_id
  61         description = get_item('description', preferred_langs)
  62         thumbnail = xpath_text(playlist, './info/thumburl', 'thumbnail')
  63         upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
  64         duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
  65         view_count = int_or_none(xpath_text(playlist, './info/views', 'views'))
  66
  67         language_preference = qualities(preferred_langs[::-1])
  68
  69         formats = []
  70         for file_ in playlist.findall('./files/file'):
  71             video_url = xpath_text(file_, './url')
  72             if not video_url:
  73                 continue
  74             lang = xpath_text(file_, './lg')
  75             formats.append({
  76                 'url': video_url,
  77                 'format_id': lang,
  78                 'format_note': xpath_text(file_, './lglabel'),
  79                 'language_preference': language_preference(lang)
  80             })
  81
  82         return {
  83             'id': video_id,
  84             'title': title,
  85             'description': description,
  86             'thumbnail': thumbnail,
  87             'upload_date': upload_date,
  88             'duration': duration,
  89             'view_count': view_count,
  90             'formats': formats
  91         }
  92
  93
  94 class EuroParlWebstreamIE(InfoExtractor):
  95     _VALID_URL = r'''(?x)
  96         https?://multimedia\.europarl\.europa\.eu/[^/#?]+/
  97         (?:(?!video)[^/#?]+/[\w-]+_)(?P<id>[\w-]+)
  98     '''
  99     _TESTS = [{
 100         'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/plenary-session_20220914-0900-PLENARY',
 101         'info_dict': {
 102             'id': '62388b15-d85b-4add-99aa-ba12ccf64f0d',
 103             'ext': 'mp4',
 104             'title': 'Plenary session',
 105             'release_timestamp': 1663139069,
 106             'release_date': '20220914',
 107         },
 108         'params': {
 109             'skip_download': True,
 110         }
 111     }, {
 112         # live webstream
 113         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/euroscola_20221115-1000-SPECIAL-EUROSCOLA',
 114         'info_dict': {
 115             'ext': 'mp4',
 116             'id': '510eda7f-ba72-161b-7ee7-0e836cd2e715',
 117             'release_timestamp': 1668502800,
 118             'title': 'Euroscola 2022-11-15 19:21',
 119             'release_date': '20221115',
 120             'live_status': 'is_live',
 121         },
 122         'skip': 'not live anymore'
 123     }, {
 124         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-culture-and-education_20230301-1130-COMMITTEE-CULT',
 125         'info_dict': {
 126             'id': '7355662c-8eac-445e-4bb9-08db14b0ddd7',
 127             'ext': 'mp4',
 128             'release_date': '20230301',
 129             'title': 'Committee on Culture and Education',
 130             'release_timestamp': 1677666641,
 131         }
 132     }, {
 133         # live stream
 134         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-environment-public-health-and-food-safety_20230524-0900-COMMITTEE-ENVI',
 135         'info_dict': {
 136             'id': 'e4255f56-10aa-4b3c-6530-08db56d5b0d9',
 137             'ext': 'mp4',
 138             'release_date': '20230524',
 139             'title': r're:Committee on Environment, Public Health and Food Safety \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}',
 140             'release_timestamp': 1684911541,
 141             'live_status': 'is_live',
 142         },
 143         'skip': 'Not live anymore'
 144     }]
 145
 146     def _real_extract(self, url):
 147         display_id = self._match_id(url)
 148         webpage = self._download_webpage(url, display_id)
 149
 150         webpage_nextjs = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
 151
 152         json_info = self._download_json(
 153             'https://acs-api.europarl.connectedviews.eu/api/FullMeeting', display_id,
 154             query={
 155                 'api-version': 1.0,
 156                 'tenantId': 'bae646ca-1fc8-4363-80ba-2c04f06b4968',
 157                 'externalReference': display_id
 158             })
 159
 160         formats, subtitles = [], {}
 161         for hls_url in traverse_obj(json_info, ((('meetingVideo'), ('meetingVideos', ...)), 'hlsUrl')):
 162             fmt, subs = self._extract_m3u8_formats_and_subtitles(hls_url, display_id)
 163             formats.extend(fmt)
 164             self._merge_subtitles(subs, target=subtitles)
 165
 166         return {
 167             'id': json_info['id'],
 168             'title': traverse_obj(webpage_nextjs, (('mediaItem', 'title'), ('title', )), get_all=False),
 169             'formats': formats,
 170             'subtitles': subtitles,
 171             'release_timestamp': parse_iso8601(json_info.get('startDateTime')),
 172             'is_live': traverse_obj(webpage_nextjs, ('mediaItem', 'mediaSubType')) == 'Live'
 173         }