yt_dlp/extractor/wikimedia.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     clean_html,
   6     get_element_by_class,
   7     parse_qs,
   8     remove_start,
   9     unescapeHTML,
  10     urljoin,
  11 )
  12
  13
  14 class WikimediaIE(InfoExtractor):
  15     IE_NAME = 'wikimedia.org'
  16     _VALID_URL = r'https?://commons\.wikimedia\.org/wiki/File:(?P<id>[^/#?]+)\.\w+'
  17     _TESTS = [{
  18         'url': 'https://commons.wikimedia.org/wiki/File:Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
  19         'info_dict': {
  20             'url': 're:https?://upload.wikimedia.org/wikipedia',
  21             'ext': 'webm',
  22             'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS',
  23             'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
  24             'description': 'md5:7cd84f76e7081f1be033d0b155b4a460',
  25             'license': 'Creative Commons Attribution 4.0 International',
  26             'uploader': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy',
  27             'subtitles': 'count:4',
  28         },
  29     }]
  30
  31     def _real_extract(self, url):
  32         video_id = self._match_id(url)
  33         webpage = self._download_webpage(url, video_id)
  34
  35         subtitles = {}
  36         for sub in set(re.findall(r'\bsrc\s*=\s*["\'](/w/api[^"]+)["\']', webpage)):
  37             sub = urljoin('https://commons.wikimedia.org', unescapeHTML(sub))
  38             qs = parse_qs(sub)
  39             lang = qs.get('lang', [None])[-1]
  40             sub_ext = qs.get('trackformat', [None])[-1]
  41             if lang and sub_ext:
  42                 subtitles.setdefault(lang, []).append({'ext': sub_ext, 'url': sub})
  43
  44         return {
  45             'id': video_id,
  46             'url': self._html_search_regex(r'<source\s[^>]*\bsrc="([^"]+)"', webpage, 'video URL'),
  47             'description': clean_html(get_element_by_class('description', webpage)),
  48             'title': remove_start(self._og_search_title(webpage), 'File:'),
  49             'license': self._html_search_regex(
  50                 r'licensed under(?: the)? (.+?) license',
  51                 get_element_by_class('licensetpl', webpage), 'license', default=None),
  52             'uploader': self._html_search_regex(
  53                 r'>\s*Author\s*</td>\s*<td\b[^>]*>\s*([^<]+)\s*</td>', webpage, 'video author', default=None),
  54             'subtitles': subtitles,
  55         }