Extracts embeds from escaped HTML within `data-html` attribute.
Related: https://github.com/ytdl-org/youtube-dl/issues/21294, https://github.com/yt-dlp/yt-dlp/pull/5121
Authored by: coletdjnz
Co-authored-by: pukkandan <redacted>
HSEShowIE,
HSEProductIE,
)
-from .genericembeds import HTML5MediaEmbedIE
+from .genericembeds import (
+ HTML5MediaEmbedIE,
+ QuotedHTMLIE,
+)
from .huajiao import HuajiaoIE
from .huya import HuyaLiveIE
from .huffpost import HuffPostIE
)
from .tv24ua import (
TV24UAVideoIE,
- TV24UAGenericPassthroughIE
)
from .tv2dk import (
TV2DKIE,
},
'playlist_count': 6,
},
- {
- # Squarespace video embed, 2019-08-28
- 'url': 'http://ootboxford.com',
- 'info_dict': {
- 'id': 'Tc7b_JGdZfw',
- 'title': 'Out of the Blue, at Childish Things 10',
- 'ext': 'mp4',
- 'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
- 'uploader_id': 'helendouglashouse',
- 'uploader': 'Helen & Douglas House',
- 'upload_date': '20140328',
- },
- 'params': {
- 'skip_download': True,
- },
- },
# {
# # Zype embed
# 'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites',
# There probably should be a second run of generic extractor on unescaped webpage.
# webpage = urllib.parse.unquote(webpage)
- # Unescape squarespace embeds to be detected by generic extractor,
- # see https://github.com/ytdl-org/youtube-dl/issues/21294
- webpage = re.sub(
- r'<div[^>]+class=[^>]*?\bsqs-video-wrapper\b[^>]*>',
- lambda x: unescapeHTML(x.group(0)), webpage)
-
# TODO: Move to respective extractors
bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
if bc_urls:
+import re
+import urllib.parse
+
from .common import InfoExtractor
-from ..utils import make_archive_id
+from ..utils import make_archive_id, unescapeHTML
class HTML5MediaEmbedIE(InfoExtractor):
})
self._sort_formats(entry['formats'])
yield entry
+
+
+class QuotedHTMLIE(InfoExtractor):
+ """For common cases of quoted/escaped html parts in the webpage"""
+ _VALID_URL = False
+ IE_NAME = 'generic:quoted-html'
+ IE_DESC = False # Do not list
+ _WEBPAGE_TESTS = [{
+ # 2 YouTube embeds in data-html
+ 'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
+ 'info_dict': {
+ 'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
+ 'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті',
+ 'thumbnail': r're:^https?://.*\.jpe?g',
+ 'timestamp': float,
+ 'upload_date': str,
+ 'description': 'md5:6816e1e5a65304bd7898e4c7eb1b26f7',
+ 'age_limit': 0,
+ },
+ 'playlist_count': 2
+ }, {
+ # Generic iframe embed of TV24UAPlayerIE within data-html
+ 'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
+ 'info_dict': {
+ 'id': '1887584',
+ 'ext': 'mp4',
+ 'title': 'Харків\'яни згадують місто до війни: щемливе відео',
+ 'thumbnail': r're:^https?://.*\.jpe?g',
+ },
+ 'params': {'skip_download': True}
+ }, {
+ # YouTube embeds on Squarespace (data-html): https://github.com/ytdl-org/youtube-dl/issues/21294
+ 'url': 'https://www.harvardballetcompany.org/past-productions',
+ 'info_dict': {
+ 'id': 'past-productions',
+ 'title': 'Productions — Harvard Ballet Company',
+ 'age_limit': 0,
+ 'description': 'Past Productions',
+ },
+ 'playlist_mincount': 26
+ }, {
+ # Squarespace video embed, 2019-08-28, data-html
+ 'url': 'http://ootboxford.com',
+ 'info_dict': {
+ 'id': 'Tc7b_JGdZfw',
+ 'title': 'Out of the Blue, at Childish Things 10',
+ 'ext': 'mp4',
+ 'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
+ 'uploader_id': 'helendouglashouse',
+ 'uploader': 'Helen & Douglas House',
+ 'upload_date': '20140328',
+ 'availability': 'public',
+ 'view_count': int,
+ 'channel': 'Helen & Douglas House',
+ 'comment_count': int,
+ 'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
+ 'duration': 253,
+ 'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
+ 'playable_in_embed': True,
+ 'age_limit': 0,
+ 'channel_follower_count': int,
+ 'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
+ 'tags': 'count:6',
+ 'categories': ['Nonprofits & Activism'],
+ 'like_count': int,
+ 'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
+
+ def _extract_from_webpage(self, url, webpage):
+ combined = ''
+ for _, html in re.findall(r'(?s)\bdata-html=(["\'])((?:(?!\1).)+)\1', webpage):
+ # unescapeHTML can handle " etc., unquote can handle percent encoding
+ unquoted_html = unescapeHTML(urllib.parse.unquote(html))
+ if unquoted_html != html:
+ combined += unquoted_html
+ if combined:
+ yield from self._extract_generic_embeds(url, combined)
-import base64
import re
-import urllib.parse
from .common import InfoExtractor
from ..utils import (
determine_ext,
- extract_attributes,
- get_elements_html_by_class,
js_to_json,
mimetype2ext,
- smuggle_url,
traverse_obj,
)
'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
'description': self._og_search_description(webpage, default=None),
}
-
-
-class TV24UAGenericPassthroughIE(InfoExtractor):
- _VALID_URL = r'https?://(?:[a-zA-Z0-9]+?\.)?24tv\.ua/(?P<id>[^/]+?_n\d+)'
-
- _TESTS = [{
- # Generic iframe, not within media_embed
- 'url': 'https://24tv.ua/vipalyuyut-nashi-mista-sela-dsns-pokazali-motoroshni-naslidki_n1883966',
- 'info_dict': {
- 'id': '1883966',
- 'ext': 'mp4',
- 'title': 'Випалюють наші міста та села, – моторошні наслідки обстрілів на Чернігівщині',
- 'thumbnail': r're:^https?://.*\.jpe?g',
- }
- }, {
- # Generic iframe embed of TV24UAPlayerIE, within media_embed
- 'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
- 'info_dict': {
- 'id': 'harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584',
- 'title': 'Харків\'яни згадують місто до війни: щемливе відео'
- },
- 'playlist': [{
- 'info_dict': {
- 'id': '1887584',
- 'ext': 'mp4',
- 'title': 'Харків\'яни згадують місто до війни: щемливе відео',
- 'thumbnail': r're:^https?://.*\.jpe?g',
- },
- }]
- }, {
- # 2 media_embeds with YouTube iframes
- 'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
- 'info_dict': {
- 'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966',
- 'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті',
- },
- 'playlist_count': 2
- }, {
- 'url': 'https://men.24tv.ua/fitnes-bloger-sprobuvav-vikonati-trenuvannya-naysilnishoyi-lyudini_n2164538',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- display_id = self._match_id(url)
- webpage = self._download_webpage(url, display_id)
- data_urls = []
- # The site contains escaped iframe embeds within an attribute.
- # Once escaped, generic can handle them, so we use a data url to pass the escaped html back.
- for html in get_elements_html_by_class('media_embed', webpage):
- data = urllib.parse.unquote(extract_attributes(html).get('data-html'))
- data_urls.append(f'data:text/html;base64,{base64.b64encode(data.encode("utf-8")).decode("utf-8")}')
-
- if not data_urls:
- return self.url_result(url, 'Generic')
- return self.playlist_from_matches(
- [smuggle_url(url, {'to_generic': True}) for url in data_urls], display_id, ie='Generic',
- playlist_title=self._og_search_title(webpage) or self._html_extract_title(webpage))
projects / yt-dlp.git / commitdiff
