diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 1dcbf71ef..8652ec54e 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -698,7 +698,10 @@ from .hse import ( HSEShowIE, HSEProductIE, ) -from .genericembeds import HTML5MediaEmbedIE +from .genericembeds import ( + HTML5MediaEmbedIE, + QuotedHTMLIE, +) from .huajiao import HuajiaoIE from .huya import HuyaLiveIE from .huffpost import HuffPostIE @@ -1884,7 +1887,6 @@ from .tv2 import ( ) from .tv24ua import ( TV24UAVideoIE, - TV24UAGenericPassthroughIE ) from .tv2dk import ( TV2DKIE, diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index ad4e3c5b8..b7a5ffb5b 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -1980,22 +1980,6 @@ class GenericIE(InfoExtractor): }, 'playlist_count': 6, }, - { - # Squarespace video embed, 2019-08-28 - 'url': 'http://ootboxford.com', - 'info_dict': { - 'id': 'Tc7b_JGdZfw', - 'title': 'Out of the Blue, at Childish Things 10', - 'ext': 'mp4', - 'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f', - 'uploader_id': 'helendouglashouse', - 'uploader': 'Helen & Douglas House', - 'upload_date': '20140328', - }, - 'params': { - 'skip_download': True, - }, - }, # { # # Zype embed # 'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites', @@ -2784,12 +2768,6 @@ class GenericIE(InfoExtractor): # There probably should be a second run of generic extractor on unescaped webpage. # webpage = urllib.parse.unquote(webpage) - # Unescape squarespace embeds to be detected by generic extractor, - # see https://github.com/ytdl-org/youtube-dl/issues/21294 - webpage = re.sub( - r']+class=[^>]*?\bsqs-video-wrapper\b[^>]*>', - lambda x: unescapeHTML(x.group(0)), webpage) - # TODO: Move to respective extractors bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage) if bc_urls: diff --git a/yt_dlp/extractor/genericembeds.py b/yt_dlp/extractor/genericembeds.py index 64bd20e3a..1bffe275a 100644 --- a/yt_dlp/extractor/genericembeds.py +++ b/yt_dlp/extractor/genericembeds.py @@ -1,5 +1,8 @@ +import re +import urllib.parse + from .common import InfoExtractor -from ..utils import make_archive_id +from ..utils import make_archive_id, unescapeHTML class HTML5MediaEmbedIE(InfoExtractor): @@ -29,3 +32,84 @@ class HTML5MediaEmbedIE(InfoExtractor): }) self._sort_formats(entry['formats']) yield entry + + +class QuotedHTMLIE(InfoExtractor): + """For common cases of quoted/escaped html parts in the webpage""" + _VALID_URL = False + IE_NAME = 'generic:quoted-html' + IE_DESC = False # Do not list + _WEBPAGE_TESTS = [{ + # 2 YouTube embeds in data-html + 'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966', + 'info_dict': { + 'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966', + 'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті', + 'thumbnail': r're:^https?://.*\.jpe?g', + 'timestamp': float, + 'upload_date': str, + 'description': 'md5:6816e1e5a65304bd7898e4c7eb1b26f7', + 'age_limit': 0, + }, + 'playlist_count': 2 + }, { + # Generic iframe embed of TV24UAPlayerIE within data-html + 'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584', + 'info_dict': { + 'id': '1887584', + 'ext': 'mp4', + 'title': 'Харків\'яни згадують місто до війни: щемливе відео', + 'thumbnail': r're:^https?://.*\.jpe?g', + }, + 'params': {'skip_download': True} + }, { + # YouTube embeds on Squarespace (data-html): https://github.com/ytdl-org/youtube-dl/issues/21294 + 'url': 'https://www.harvardballetcompany.org/past-productions', + 'info_dict': { + 'id': 'past-productions', + 'title': 'Productions — Harvard Ballet Company', + 'age_limit': 0, + 'description': 'Past Productions', + }, + 'playlist_mincount': 26 + }, { + # Squarespace video embed, 2019-08-28, data-html + 'url': 'http://ootboxford.com', + 'info_dict': { + 'id': 'Tc7b_JGdZfw', + 'title': 'Out of the Blue, at Childish Things 10', + 'ext': 'mp4', + 'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f', + 'uploader_id': 'helendouglashouse', + 'uploader': 'Helen & Douglas House', + 'upload_date': '20140328', + 'availability': 'public', + 'view_count': int, + 'channel': 'Helen & Douglas House', + 'comment_count': int, + 'uploader_url': 'http://www.youtube.com/user/helendouglashouse', + 'duration': 253, + 'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA', + 'playable_in_embed': True, + 'age_limit': 0, + 'channel_follower_count': int, + 'channel_id': 'UCTChGezrZVmlYlpMlkmulPA', + 'tags': 'count:6', + 'categories': ['Nonprofits & Activism'], + 'like_count': int, + 'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg', + }, + 'params': { + 'skip_download': True, + }, + }] + + def _extract_from_webpage(self, url, webpage): + combined = '' + for _, html in re.findall(r'(?s)\bdata-html=(["\'])((?:(?!\1).)+)\1', webpage): + # unescapeHTML can handle " etc., unquote can handle percent encoding + unquoted_html = unescapeHTML(urllib.parse.unquote(html)) + if unquoted_html != html: + combined += unquoted_html + if combined: + yield from self._extract_generic_embeds(url, combined) diff --git a/yt_dlp/extractor/tv24ua.py b/yt_dlp/extractor/tv24ua.py index 723049e78..553a70b6b 100644 --- a/yt_dlp/extractor/tv24ua.py +++ b/yt_dlp/extractor/tv24ua.py @@ -1,15 +1,10 @@ -import base64 import re -import urllib.parse from .common import InfoExtractor from ..utils import ( determine_ext, - extract_attributes, - get_elements_html_by_class, js_to_json, mimetype2ext, - smuggle_url, traverse_obj, ) @@ -87,60 +82,3 @@ class TV24UAVideoIE(InfoExtractor): 'title': self._html_extract_title(webpage) or self._og_search_title(webpage), 'description': self._og_search_description(webpage, default=None), } - - -class TV24UAGenericPassthroughIE(InfoExtractor): - _VALID_URL = r'https?://(?:[a-zA-Z0-9]+?\.)?24tv\.ua/(?P[^/]+?_n\d+)' - - _TESTS = [{ - # Generic iframe, not within media_embed - 'url': 'https://24tv.ua/vipalyuyut-nashi-mista-sela-dsns-pokazali-motoroshni-naslidki_n1883966', - 'info_dict': { - 'id': '1883966', - 'ext': 'mp4', - 'title': 'Випалюють наші міста та села, – моторошні наслідки обстрілів на Чернігівщині', - 'thumbnail': r're:^https?://.*\.jpe?g', - } - }, { - # Generic iframe embed of TV24UAPlayerIE, within media_embed - 'url': 'https://24tv.ua/harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584', - 'info_dict': { - 'id': 'harkivyani-zgaduyut-misto-do-viyni-shhemlive-video_n1887584', - 'title': 'Харків\'яни згадують місто до війни: щемливе відео' - }, - 'playlist': [{ - 'info_dict': { - 'id': '1887584', - 'ext': 'mp4', - 'title': 'Харків\'яни згадують місто до війни: щемливе відео', - 'thumbnail': r're:^https?://.*\.jpe?g', - }, - }] - }, { - # 2 media_embeds with YouTube iframes - 'url': 'https://24tv.ua/bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966', - 'info_dict': { - 'id': 'bronetransporteri-ozbroyenni-zsu-shho-vidomo-pro-bronovik-wolfhound_n2167966', - 'title': 'Броньовик Wolfhound: гігант, який допомагає ЗСУ знищувати окупантів на фронті', - }, - 'playlist_count': 2 - }, { - 'url': 'https://men.24tv.ua/fitnes-bloger-sprobuvav-vikonati-trenuvannya-naysilnishoyi-lyudini_n2164538', - 'only_matching': True, - }] - - def _real_extract(self, url): - display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - data_urls = [] - # The site contains escaped iframe embeds within an attribute. - # Once escaped, generic can handle them, so we use a data url to pass the escaped html back. - for html in get_elements_html_by_class('media_embed', webpage): - data = urllib.parse.unquote(extract_attributes(html).get('data-html')) - data_urls.append(f'data:text/html;base64,{base64.b64encode(data.encode("utf-8")).decode("utf-8")}') - - if not data_urls: - return self.url_result(url, 'Generic') - return self.playlist_from_matches( - [smuggle_url(url, {'to_generic': True}) for url in data_urls], display_id, ie='Generic', - playlist_title=self._og_search_title(webpage) or self._html_extract_title(webpage))