yt_dlp/extractor/genericembeds.py

   1 import re
   2 import urllib.parse
   3
   4 from .common import InfoExtractor
   5 from ..utils import make_archive_id, unescapeHTML
   6
   7
   8 class HTML5MediaEmbedIE(InfoExtractor):
   9     _VALID_URL = False
  10     IE_NAME = 'html5'
  11     _WEBPAGE_TESTS = [
  12         {
  13             'url': 'https://html.com/media/',
  14             'info_dict': {
  15                 'title': 'HTML5 Media',
  16                 'description': 'md5:933b2d02ceffe7a7a0f3c8326d91cc2a',
  17             },
  18             'playlist_count': 2
  19         }
  20     ]
  21
  22     def _extract_from_webpage(self, url, webpage):
  23         video_id, title = self._generic_id(url), self._generic_title(url)
  24         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls') or []
  25         for num, entry in enumerate(entries, start=1):
  26             entry.update({
  27                 'id': f'{video_id}-{num}',
  28                 'title': f'{title} ({num})',
  29                 '_old_archive_ids': [
  30                     make_archive_id('generic', f'{video_id}-{num}' if len(entries) > 1 else video_id),
  31                 ],
  32             })
  33             self._sort_formats(entry['formats'])
  34             yield entry
  35
  36
  37 class QuotedHTMLIE(InfoExtractor):
  38     """For common cases of quoted/escaped html parts in the webpage"""
  39     _VALID_URL = False
  40     IE_NAME = 'generic:quoted-html'
  41     IE_DESC = False  # Do not list
  42     _WEBPAGE_TESTS = [{
  43         # 2 YouTube embeds in data-html
  44         'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
  45         'info_dict': {
  46             'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
  47             'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті',
  48             'thumbnail': r're:^https?://.*\.jpe?g',
  49             'timestamp': float,
  50             'upload_date': str,
  51             'description': 'md5:6816e1e5a65304bd7898e4c7eb1b26f7',
  52             'age_limit': 0,
  53         },
  54         'playlist_count': 2
  55     }, {
  56         # Generic iframe embed of TV24UAPlayerIE within data-html
  57         'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
  58         'info_dict': {
  59             'id': '1887584',
  60             'ext': 'mp4',
  61             'title': 'Харків\'яни згадують місто до війни: щемливе відео',
  62             'thumbnail': r're:^https?://.*\.jpe?g',
  63         },
  64         'params': {'skip_download': True}
  65     }, {
  66         # YouTube embeds on Squarespace (data-html): https://github.com/ytdl-org/youtube-dl/issues/21294
  67         'url': 'https://www.harvardballetcompany.org/past-productions',
  68         'info_dict': {
  69             'id': 'past-productions',
  70             'title': 'Productions — Harvard Ballet Company',
  71             'age_limit': 0,
  72             'description': 'Past Productions',
  73         },
  74         'playlist_mincount': 26
  75     }, {
  76         # Squarespace video embed, 2019-08-28, data-html
  77         'url': 'http://ootboxford.com',
  78         'info_dict': {
  79             'id': 'Tc7b_JGdZfw',
  80             'title': 'Out of the Blue, at Childish Things 10',
  81             'ext': 'mp4',
  82             'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
  83             'uploader_id': 'helendouglashouse',
  84             'uploader': 'Helen & Douglas House',
  85             'upload_date': '20140328',
  86             'availability': 'public',
  87             'view_count': int,
  88             'channel': 'Helen & Douglas House',
  89             'comment_count': int,
  90             'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
  91             'duration': 253,
  92             'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
  93             'playable_in_embed': True,
  94             'age_limit': 0,
  95             'channel_follower_count': int,
  96             'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
  97             'tags': 'count:6',
  98             'categories': ['Nonprofits & Activism'],
  99             'like_count': int,
 100             'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
 101         },
 102         'params': {
 103             'skip_download': True,
 104         },
 105     }]
 106
 107     def _extract_from_webpage(self, url, webpage):
 108         combined = ''
 109         for _, html in re.findall(r'(?s)\bdata-html=(["\'])((?:(?!\1).)+)\1', webpage):
 110             # unescapeHTML can handle &quot; etc., unquote can handle percent encoding
 111             unquoted_html = unescapeHTML(urllib.parse.unquote(html))
 112             if unquoted_html != html:
 113                 combined += unquoted_html
 114         if combined:
 115             yield from self._extract_generic_embeds(url, combined)